diff --git a/.codecov.yml b/.codecov.yml index ed122151c1..1d3856c7f1 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -5,6 +5,7 @@ coverage: ignore: - "**/*_test.go" - "**/enum_*.go" + - "./cmd/create-s3-replication-job/main.go" - "./cmd/event-logger/main.go" - "./cmd/event-received/main.go" - "./cmd/mlpa/main.go" diff --git a/.github/workflows/docker_job.yml b/.github/workflows/docker_job.yml index fcf54a4c7c..271a257714 100644 --- a/.github/workflows/docker_job.yml +++ b/.github/workflows/docker_job.yml @@ -40,8 +40,8 @@ jobs: platforms: linux/amd64 - ecr_repository: modernising-lpa/create-s3-batch-replication-job name: create-s3-batch-replication-job - path: ./lambda/create_s3_replication_job/Dockerfile - trivyignores: ./lambda/create_s3_replication_job/.trivyignore.yaml + path: ./docker/create-s3-replication-job/Dockerfile + trivyignores: ./docker/create-s3-replication-job/.trivyignore.yaml platforms: linux/amd64 - ecr_repository: modernising-lpa/event-received name: event-received diff --git a/cmd/create-s3-replication-job/main.go b/cmd/create-s3-replication-job/main.go new file mode 100644 index 0000000000..736a016273 --- /dev/null +++ b/cmd/create-s3-replication-job/main.go @@ -0,0 +1,154 @@ +// Create S3 replication job is an AWS Lambda function used to create an S3 +// Batch Replication Job to copy files from one S3 bucket to another. +// +// In this service, the source bucket is for uploads to the service and the +// destination bucket is for a case management system in another AWS account. +package main + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + + "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3control" + "github.com/aws/aws-sdk-go-v2/service/s3control/types" + "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/google/uuid" + "github.com/ministryofjustice/opg-modernising-lpa/internal/telemetry" + "go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda" + "go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda/xrayconfig" +) + +var ( + environment = os.Getenv("ENVIRONMENT") + logger *slog.Logger + cfg aws.Config +) + +type configVars struct { + AccountID string `json:"aws_account_id"` + Environment string `json:'-"` + ReportAndManifestsBucket string `json:"report_and_manifests_bucket"` + RoleARN string `json:"role_arn"` + SourceBucket string `json:"source_bucket"` +} + +func main() { + ctx := context.Background() + + logger = slog.New(telemetry.NewSlogHandler(slog. + NewJSONHandler(os.Stdout, nil)). + WithAttrs([]slog.Attr{ + slog.String("service_name", "opg-modernising-lpa/create-s3-replication-job"), + })) + + var err error + cfg, err = config.LoadDefaultConfig(ctx) + if err != nil { + logger.ErrorContext(ctx, "failed to load default config", slog.Any("err", err)) + return + } + + tp, err := telemetry.SetupLambda(ctx, &cfg.APIOptions) + if err != nil { + logger.WarnContext(ctx, "error creating tracer provider", slog.Any("err", err)) + } + + if tp != nil { + defer func(ctx context.Context) { + if err := tp.Shutdown(ctx); err != nil { + logger.WarnContext(ctx, "error shutting down tracer provider", slog.Any("err", err)) + } + }(ctx) + + lambda.Start(otellambda.InstrumentHandler(handler, xrayconfig.WithRecommendedOptions(tp)...)) + } else { + lambda.Start(handler) + } +} + +func handler(ctx context.Context) error { + vars, err := getVars(ctx, cfg, environment) + if err != nil { + return fmt.Errorf("failed to get config vars: %w", err) + } + + jobID, err := createJob(ctx, cfg, vars) + if err != nil { + return fmt.Errorf("failed to create job: %w", err) + } + + logger.InfoContext(ctx, "job created", slog.Any("job_id", jobID)) + return nil +} + +func getVars(ctx context.Context, cfg aws.Config, environment string) (configVars, error) { + ssmClient := ssm.NewFromConfig(cfg) + + param, err := ssmClient.GetParameter(ctx, &ssm.GetParameterInput{ + Name: aws.String("/modernising-lpa/s3-batch-configuration/" + environment + "/s3_batch_configuration"), + }) + if err != nil { + return configVars{}, fmt.Errorf("failed to retrieve parameter: %w", err) + } + + var vars configVars + if err := json.Unmarshal([]byte(*param.Parameter.Value), &vars); err != nil { + return configVars{}, fmt.Errorf("failed to unmarshal parameter: %w", err) + } + + vars.Environment = environment + return vars, nil +} + +func createJob(ctx context.Context, cfg aws.Config, vars configVars) (string, error) { + controlClient := s3control.NewFromConfig(cfg) + requestToken := uuid.NewString() + + resp, err := controlClient.CreateJob(ctx, &s3control.CreateJobInput{ + AccountId: aws.String(vars.AccountID), + ConfirmationRequired: aws.Bool(false), + Operation: &types.JobOperation{ + S3ReplicateObject: &types.S3ReplicateObjectOperation{}, + }, + Report: &types.JobReport{ + Enabled: true, + Bucket: aws.String(vars.ReportAndManifestsBucket), + Format: types.JobReportFormatReportCsv20180820, + ReportScope: types.JobReportScopeAllTasks, + }, + ClientRequestToken: aws.String(requestToken), + Description: aws.String("S3 replication " + vars.Environment + " - golang"), + Priority: aws.Int32(10), + RoleArn: aws.String(vars.RoleARN), + ManifestGenerator: &types.JobManifestGeneratorMemberS3JobManifestGenerator{ + Value: types.S3JobManifestGenerator{ + EnableManifestOutput: true, + ExpectedBucketOwner: aws.String(vars.AccountID), + SourceBucket: aws.String(vars.SourceBucket), + Filter: &types.JobManifestGeneratorFilter{ + EligibleForReplication: aws.Bool(true), + ObjectReplicationStatuses: []types.ReplicationStatus{types.ReplicationStatusFailed, types.ReplicationStatusNone}, + }, + ManifestOutputLocation: &types.S3ManifestOutputLocation{ + ExpectedManifestBucketOwner: aws.String(vars.AccountID), + Bucket: aws.String(vars.ReportAndManifestsBucket), + ManifestEncryption: &types.GeneratedManifestEncryption{ + SSES3: &types.SSES3Encryption{}, + }, + ManifestFormat: types.GeneratedManifestFormatS3InventoryReportCsv20211130, + }, + }, + }, + }) + if err != nil { + return "", err + } + + return *resp.JobId, nil +} diff --git a/cmd/event-received/main.go b/cmd/event-received/main.go index 755cd1c82e..e49a31ecff 100644 --- a/cmd/event-received/main.go +++ b/cmd/event-received/main.go @@ -26,7 +26,6 @@ import ( "github.com/ministryofjustice/opg-modernising-lpa/internal/telemetry" "go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda" "go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda/xrayconfig" - "go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel/sdk/trace" ) @@ -246,15 +245,13 @@ func main() { var tp *trace.TracerProvider if xrayEnabled { - tp, err = telemetry.SetupLambda(ctx) + tp, err = telemetry.SetupLambda(ctx, &cfg.APIOptions) if err != nil { logger.WarnContext(ctx, "error creating tracer provider", slog.Any("err", err)) } } if tp != nil { - otelaws.AppendMiddlewares(&cfg.APIOptions) - telemetry.AppendMiddlewares(&cfg.APIOptions) httpClient.Transport = otelhttp.NewTransport(httpClient.Transport) defer func(ctx context.Context) { diff --git a/cmd/mlpa/main.go b/cmd/mlpa/main.go index d7e441b6ff..d2714e0d56 100644 --- a/cmd/mlpa/main.go +++ b/cmd/mlpa/main.go @@ -41,7 +41,6 @@ import ( "github.com/ministryofjustice/opg-modernising-lpa/internal/templatefn" "github.com/ministryofjustice/opg-modernising-lpa/internal/uid" "go.opentelemetry.io/contrib/detectors/aws/ecs" - "go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "golang.org/x/mod/sumdb/dirhash" ) @@ -124,13 +123,22 @@ func run(ctx context.Context, logger *slog.Logger) error { httpClient := &http.Client{Timeout: 30 * time.Second} + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return fmt.Errorf("unable to load SDK config: %w", err) + } + + if len(awsBaseURL) > 0 { + cfg.BaseEndpoint = aws.String(awsBaseURL) + } + if xrayEnabled { resource, err := ecs.NewResourceDetector().Detect(ctx) if err != nil { return err } - shutdown, err := telemetry.Setup(ctx, resource) + shutdown, err := telemetry.Setup(ctx, resource, &cfg.APIOptions) if err != nil { return err } @@ -195,18 +203,6 @@ func run(ctx context.Context, logger *slog.Logger) error { return err } - cfg, err := config.LoadDefaultConfig(ctx) - if err != nil { - return fmt.Errorf("unable to load SDK config: %w", err) - } - - if len(awsBaseURL) > 0 { - cfg.BaseEndpoint = aws.String(awsBaseURL) - } - - otelaws.AppendMiddlewares(&cfg.APIOptions) - telemetry.AppendMiddlewares(&cfg.APIOptions) - lpasDynamoClient, err := dynamo.NewClient(cfg, dynamoTableLpas) if err != nil { return err diff --git a/cmd/schedule-runner/main.go b/cmd/schedule-runner/main.go index 7639be700c..60b09f4ab9 100644 --- a/cmd/schedule-runner/main.go +++ b/cmd/schedule-runner/main.go @@ -23,7 +23,6 @@ import ( "github.com/ministryofjustice/opg-modernising-lpa/internal/telemetry" "go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda" "go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda/xrayconfig" - "go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel/sdk/trace" ) @@ -128,15 +127,13 @@ func main() { var tp *trace.TracerProvider if xrayEnabled { - tp, err = telemetry.SetupLambda(ctx) + tp, err = telemetry.SetupLambda(ctx, &cfg.APIOptions) if err != nil { logger.WarnContext(ctx, "error creating tracer provider", slog.Any("err", err)) } } if tp != nil { - otelaws.AppendMiddlewares(&cfg.APIOptions) - telemetry.AppendMiddlewares(&cfg.APIOptions) httpClient.Transport = otelhttp.NewTransport(httpClient.Transport) defer func(ctx context.Context) { diff --git a/lambda/create_s3_replication_job/.trivyignore.yaml b/docker/create-s3-replication-job/.trivyignore.yaml similarity index 100% rename from lambda/create_s3_replication_job/.trivyignore.yaml rename to docker/create-s3-replication-job/.trivyignore.yaml diff --git a/docker/create-s3-replication-job/Dockerfile b/docker/create-s3-replication-job/Dockerfile new file mode 100644 index 0000000000..ce454f92da --- /dev/null +++ b/docker/create-s3-replication-job/Dockerfile @@ -0,0 +1,44 @@ +FROM golang:1.23.2-alpine AS build + +WORKDIR /app + +COPY --link go.mod go.sum ./ +RUN go mod download + +COPY --link cmd/create-s3-replication-job ./cmd/create-s3-replication-job +COPY --link internal ./internal + +RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -o create-s3-replication-job ./cmd/create-s3-replication-job + +FROM public.ecr.aws/lambda/provided:al2023.2024.10.14.12 AS dev + +WORKDIR /app + +COPY --from=build /app/create-s3-replication-job /var/task/create-s3-replication-job +COPY --link lang ./lang +COPY --link ./docker/adot-collector/ /opt +COPY --link docker/aws-lambda-rie ./aws-lambda-rie + +ENV AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler +ENV OPENTELEMETRY_COLLECTOR_CONFIG_FILE="/opt/config/config.yaml" + +ENTRYPOINT ["./create-s3-replication-job"] + +FROM public.ecr.aws/lambda/provided:al2023.2024.10.14.12 AS production + +WORKDIR /app +COPY --link docker/install_lambda_insights.sh /app/ + +RUN chmod +x "/app/install_lambda_insights.sh" \ + && /app/install_lambda_insights.sh "${TARGETPLATFORM}" + +COPY --from=build /app/create-s3-replication-job ./create-s3-replication-job +COPY --link lang ./lang +COPY --link ./docker/adot-collector/ /opt + +RUN chmod 755 /opt/config/config.yaml + +ENV AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler +ENV OPENTELEMETRY_COLLECTOR_CONFIG_FILE="/opt/config/config.yaml" + +ENTRYPOINT ["./create-s3-replication-job"] diff --git a/go.mod b/go.mod index dc73aec66d..ed63f44983 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/MicahParks/jwkset v0.5.20 github.com/MicahParks/keyfunc/v3 v3.3.5 github.com/aws/aws-lambda-go v1.47.0 - github.com/aws/aws-sdk-go-v2 v1.32.3 + github.com/aws/aws-sdk-go-v2 v1.32.4 github.com/aws/aws-sdk-go-v2/config v1.28.1 github.com/aws/aws-sdk-go-v2/credentials v1.17.42 github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.13 @@ -49,8 +49,8 @@ require ( require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.24.3 // indirect @@ -58,7 +58,9 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4 // indirect + github.com/aws/aws-sdk-go-v2/service/s3control v1.50.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssm v1.55.4 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 // indirect diff --git a/go.sum b/go.sum index f0ec8c5d98..18e9a96db5 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcT github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk= github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2 v1.32.4 h1:S13INUiTxgrPueTmrm5DZ+MiAo99zYzHEFh1UNkOxNE= +github.com/aws/aws-sdk-go-v2 v1.32.4/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= github.com/aws/aws-sdk-go-v2/config v1.28.0 h1:FosVYWcqEtWNxHn8gB/Vs6jOlNwSoyOCA/g/sxyySOQ= @@ -30,10 +32,14 @@ github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79Wzp github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 h1:A2w6m6Tmr+BNXjDsr7M90zkWjsu4JXHwrzPg235STs4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23/go.mod h1:35EVp9wyeANdujZruvHiQUAo9E3vbhnIO1mTCAxMlY0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 h1:pgYW9FCabt2M25MoHYCfMrVY2ghiiBKYWUVXfwZs+sU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23/go.mod h1:c48kLgzO19wAu3CPkDWC28JbaJ+hfQlsdl7I2+oqIbk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 h1:7edmS3VOBDhK00b/MwGtGglCm7hhwNYnjJs/PgFdMQE= @@ -70,12 +76,16 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2/go.mod h1:/niFCtmuQNxqx9v8WAPq5qh7EH25U4BF6tjoyq9bObM= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 h1:ZC7Y/XgKUxwqcdhO5LE8P6oGP1eh6xlQReWNKfhvJno= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3/go.mod h1:WqfO7M9l9yUAw0HcHaikwRd/H6gzYdz7vjejCA5e2oY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4 h1:E5ZAVOmI2apR8ADb72Q63KqwwwdW1XcMeXIlrZ1Psjg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4/go.mod h1:wezzqVUOVVdk+2Z/JzQT4NxAU0NbhRe5W8pIE72jsWI= github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0 h1:xA6XhTF7PE89BCNHJbQi8VvPzcgMtmGC5dr8S8N7lHk= github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= github.com/aws/aws-sdk-go-v2/service/s3 v1.66.1 h1:MkQ4unegQEStiQYmfFj+Aq5uTp265ncSmm0XTQwDwi0= github.com/aws/aws-sdk-go-v2/service/s3 v1.66.1/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2 h1:p9TNFL8bFUMd+38YIpTAXpoxyz0MxC7FlbFEH4P4E1U= github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2/go.mod h1:fNjyo0Coen9QTwQLWeV6WO2Nytwiu+cCcWaTdKCAqqE= +github.com/aws/aws-sdk-go-v2/service/s3control v1.50.0 h1:sCBj2KHGcbp9rZZ2HhwGSozndlZPnfZlidPKAdGySlo= +github.com/aws/aws-sdk-go-v2/service/s3control v1.50.0/go.mod h1:5Jtme+EepIEeN+icvNwSG+4GN7WPF9Q3AiBT9sLquEY= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2 h1:Rrqru2wYkKQCS2IM5/JrgKUQIoNTqA6y/iuxkjzxC6M= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2/go.mod h1:QuCURO98Sqee2AXmqDNxKXYFm2OEDAVAPApMqO0Vqnc= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.3 h1:CyA6J82ePPoh1Nj8ErOR2e/JRlzfFzWpGwGMFzFjwZg= @@ -84,6 +94,8 @@ github.com/aws/aws-sdk-go-v2/service/sqs v1.36.2 h1:kmbcoWgbzfh5a6rvfjOnfHSGEqD1 github.com/aws/aws-sdk-go-v2/service/sqs v1.36.2/go.mod h1:/UPx74a3M0WYeT2yLQYG/qHhkPlPXd6TsppfGgy2COk= github.com/aws/aws-sdk-go-v2/service/sqs v1.36.3 h1:H1bCg79Q4PDtxQH8Fn5kASQlbVv2WGP5o5IEFEBNOAs= github.com/aws/aws-sdk-go-v2/service/sqs v1.36.3/go.mod h1:W6Uy6OWgxF9RZuHoikthB6f+A0oYXqnfWmFl5m7E2G4= +github.com/aws/aws-sdk-go-v2/service/ssm v1.55.4 h1:CcUHAf22CEhJ+GpTYIsbXmVqWATzC3FSxekg/IWoi4E= +github.com/aws/aws-sdk-go-v2/service/ssm v1.55.4/go.mod h1:zH7gDT/mAjLk10jcoltSXvjruPmvDSpfCTqzA+0B3l4= github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk= github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 h1:UTpsIf0loCIWEbrqdLb+0RxnTXfWh2vhw4nQmFi4nPc= diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index 1b2f228f66..ee218e1603 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -8,8 +8,10 @@ import ( "net/http" "strings" + "github.com/aws/smithy-go/middleware" "github.com/felixge/httpsnoop" "go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda/xrayconfig" + "go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws" "go.opentelemetry.io/contrib/propagators/aws/xray" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -20,7 +22,7 @@ import ( "go.opentelemetry.io/otel/trace" ) -func Setup(ctx context.Context, resource *resource.Resource) (func(context.Context) error, error) { +func Setup(ctx context.Context, resource *resource.Resource, apiOptions *[]func(*middleware.Stack) error) (func(context.Context) error, error) { exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure(), otlptracegrpc.WithEndpoint("0.0.0.0:4317")) @@ -41,12 +43,15 @@ func Setup(ctx context.Context, resource *resource.Resource) (func(context.Conte otel.SetTracerProvider(tp) otel.SetTextMapPropagator(xray.Propagator{}) + otelaws.AppendMiddlewares(apiOptions) + AppendMiddlewares(apiOptions) + return tp.Shutdown, nil } // SetupLambda requires an ADOT collector lambda extension to be included in the // deployed lambda image for the below instrumentation to work -func SetupLambda(ctx context.Context) (*sdktrace.TracerProvider, error) { +func SetupLambda(ctx context.Context, apiOptions *[]func(*middleware.Stack) error) (*sdktrace.TracerProvider, error) { tp, err := xrayconfig.NewTracerProvider(ctx) if err != nil { return nil, err @@ -55,6 +60,9 @@ func SetupLambda(ctx context.Context) (*sdktrace.TracerProvider, error) { otel.SetTracerProvider(tp) otel.SetTextMapPropagator(xray.Propagator{}) + otelaws.AppendMiddlewares(apiOptions) + AppendMiddlewares(apiOptions) + return tp, nil } diff --git a/lambda/create_s3_replication_job/Dockerfile b/lambda/create_s3_replication_job/Dockerfile deleted file mode 100644 index 7f0b23e011..0000000000 --- a/lambda/create_s3_replication_job/Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -# Define function directory -ARG FUNCTION_DIR="/function" - -FROM python:3.13-alpine3.19 AS python-alpine -RUN apk add --no-cache \ - libstdc++=13.2.1_git20231014-r0 \ - elfutils-dev=0.190-r1 - -FROM python-alpine AS build-image -# Install aws-lambda-cpp build dependencies -RUN apk add --no-cache \ - build-base=0.5-r3 \ - libtool=2.4.7-r3 \ - autoconf=2.71-r2 \ - automake=1.16.5-r2 \ - make=4.4.1-r2 \ - cmake=3.27.8-r0 \ - libcurl=8.9.1-r1 - -# Include global arg in this stage of the build -ARG FUNCTION_DIR -# Create function directory and set working directory to function root directory -RUN mkdir -p ${FUNCTION_DIR} -WORKDIR ${FUNCTION_DIR} - -# Copy function code and requirements.txt -COPY lambda/create_s3_replication_job/src/main.py lambda/create_s3_replication_job/src/requirements.txt ${FUNCTION_DIR}/ - -# Install the runtime interface client -RUN python -m pip install --root-user-action=ignore \ - --no-cache-dir \ - --target ${FUNCTION_DIR} \ - --requirement requirements.txt - -# Multi-stage build: grab a fresh copy of the base image -FROM python-alpine AS production -# Include global arg in this stage of the build -ARG FUNCTION_DIR -# Set working directory to function root directory -WORKDIR ${FUNCTION_DIR} - -# Copy in the build image dependencies -COPY --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR} - -ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] -CMD ["main.handler"] diff --git a/lambda/create_s3_replication_job/README.md b/lambda/create_s3_replication_job/README.md deleted file mode 100644 index 32e78498ad..0000000000 --- a/lambda/create_s3_replication_job/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Creat S3 Batch Replication Job Lambda - -This lambda is used to create a S3 Batch Replication Job to copy files from one S3 bucket to another. - -In this service, the source bucket is for uploads to the service and the destination bucket is for a case management system in another AWS account. diff --git a/lambda/create_s3_replication_job/src/main.py b/lambda/create_s3_replication_job/src/main.py deleted file mode 100644 index 42005864d3..0000000000 --- a/lambda/create_s3_replication_job/src/main.py +++ /dev/null @@ -1,93 +0,0 @@ -import os -import logging -import uuid -import json -import boto3 -from aws_xray_sdk.core import patch_all, xray_recorder - -logger = logging.getLogger() -logger.setLevel(logging.INFO) -xray_recorder.begin_segment('reduced_fees_uploads') -patch_all() - - -def handler(event, context): - subsegment = xray_recorder.begin_subsegment('create_s3_batch_replication_job') - subsegment.put_annotation('service', 'reduced_fees_uploads') - variables = set_variables() - client = create_client() - response = create_s3_batch_replication_job(client, variables) - logger.info(response) - xray_recorder.end_subsegment() - -def main(): - variables = set_variables() - client = create_client() - response = create_s3_batch_replication_job(client, variables) - print(f"job ID: {response['JobId']}") - logger.info(response) - -def create_client(): - client = boto3.client( - 's3control', - region_name='eu-west-1' - ) - return client - -def set_variables(): - environment = os.getenv('ENVIRONMENT') - ssm_client = boto3.client('ssm') - parameter = ssm_client.get_parameter( - Name=f'/modernising-lpa/s3-batch-configuration/{environment}/s3_batch_configuration', - ) - json_object = json.loads(parameter['Parameter']['Value']) - json_object.update({"environment": environment}) - return json_object - - -def create_s3_batch_replication_job(client, variables): - request_token = str(uuid.uuid4()) - response = client.create_job( - AccountId=variables['aws_account_id'], - ConfirmationRequired=False, - Operation={ - 'S3ReplicateObject': {} - }, - Report={ - 'Bucket': variables['report_and_manifests_bucket'], - 'Format': 'Report_CSV_20180820', - 'Enabled': True, - 'ReportScope': 'AllTasks' - }, - ClientRequestToken=request_token, - - Description=f'S3 replication {variables["environment"]} - python', - Priority=10, - RoleArn=variables['role_arn'], - ManifestGenerator={ - 'S3JobManifestGenerator': { - 'ExpectedBucketOwner': variables['aws_account_id'], - 'SourceBucket': variables['source_bucket'], - 'ManifestOutputLocation': { - 'ExpectedManifestBucketOwner': variables['aws_account_id'], - 'Bucket': variables['report_and_manifests_bucket'], - 'ManifestEncryption': { - 'SSES3': {} - }, - 'ManifestFormat': 'S3InventoryReport_CSV_20211130' - }, - 'Filter': { - 'EligibleForReplication': True, - 'ObjectReplicationStatuses': [ - 'FAILED', - 'NONE' - ] - }, - 'EnableManifestOutput': True - } - } - ) - return response - -if __name__ == '__main__': - main() diff --git a/lambda/create_s3_replication_job/src/requirements.txt b/lambda/create_s3_replication_job/src/requirements.txt deleted file mode 100644 index ac0e082674..0000000000 --- a/lambda/create_s3_replication_job/src/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -requests==2.32.3 -boto3==1.35.53 -aws-xray-sdk==2.14.0 -awslambdaric==2.2.1