-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1605 from ministryofjustice/MLPAB-2633-replication
MLPAB-2633 Rewrite create-s3-replication-job to golang
- Loading branch information
Showing
15 changed files
with
241 additions
and
178 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,154 @@ | ||
| // Create S3 replication job is an AWS Lambda function used to create an S3 | ||
| // Batch Replication Job to copy files from one S3 bucket to another. | ||
| // | ||
| // In this service, the source bucket is for uploads to the service and the | ||
| // destination bucket is for a case management system in another AWS account. | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
| "log/slog" | ||
| "os" | ||
|
|
||
| "github.com/aws/aws-lambda-go/lambda" | ||
| "github.com/aws/aws-sdk-go-v2/aws" | ||
| "github.com/aws/aws-sdk-go-v2/config" | ||
| "github.com/aws/aws-sdk-go-v2/service/s3control" | ||
| "github.com/aws/aws-sdk-go-v2/service/s3control/types" | ||
| "github.com/aws/aws-sdk-go-v2/service/ssm" | ||
| "github.com/google/uuid" | ||
| "github.com/ministryofjustice/opg-modernising-lpa/internal/telemetry" | ||
| "go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda" | ||
| "go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda/xrayconfig" | ||
| ) | ||
|
|
||
| var ( | ||
| environment = os.Getenv("ENVIRONMENT") | ||
| logger *slog.Logger | ||
| cfg aws.Config | ||
| ) | ||
|
|
||
| type configVars struct { | ||
| AccountID string `json:"aws_account_id"` | ||
| Environment string `json:'-"` | ||
| ReportAndManifestsBucket string `json:"report_and_manifests_bucket"` | ||
| RoleARN string `json:"role_arn"` | ||
| SourceBucket string `json:"source_bucket"` | ||
| } | ||
|
|
||
| func main() { | ||
| ctx := context.Background() | ||
|
|
||
| logger = slog.New(telemetry.NewSlogHandler(slog. | ||
| NewJSONHandler(os.Stdout, nil)). | ||
| WithAttrs([]slog.Attr{ | ||
| slog.String("service_name", "opg-modernising-lpa/create-s3-replication-job"), | ||
| })) | ||
|
|
||
| var err error | ||
| cfg, err = config.LoadDefaultConfig(ctx) | ||
| if err != nil { | ||
| logger.ErrorContext(ctx, "failed to load default config", slog.Any("err", err)) | ||
| return | ||
| } | ||
|
|
||
| tp, err := telemetry.SetupLambda(ctx, &cfg.APIOptions) | ||
| if err != nil { | ||
| logger.WarnContext(ctx, "error creating tracer provider", slog.Any("err", err)) | ||
| } | ||
|
|
||
| if tp != nil { | ||
| defer func(ctx context.Context) { | ||
| if err := tp.Shutdown(ctx); err != nil { | ||
| logger.WarnContext(ctx, "error shutting down tracer provider", slog.Any("err", err)) | ||
| } | ||
| }(ctx) | ||
|
|
||
| lambda.Start(otellambda.InstrumentHandler(handler, xrayconfig.WithRecommendedOptions(tp)...)) | ||
| } else { | ||
| lambda.Start(handler) | ||
| } | ||
| } | ||
|
|
||
| func handler(ctx context.Context) error { | ||
| vars, err := getVars(ctx, cfg, environment) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to get config vars: %w", err) | ||
| } | ||
|
|
||
| jobID, err := createJob(ctx, cfg, vars) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to create job: %w", err) | ||
| } | ||
|
|
||
| logger.InfoContext(ctx, "job created", slog.Any("job_id", jobID)) | ||
| return nil | ||
| } | ||
|
|
||
| func getVars(ctx context.Context, cfg aws.Config, environment string) (configVars, error) { | ||
| ssmClient := ssm.NewFromConfig(cfg) | ||
|
|
||
| param, err := ssmClient.GetParameter(ctx, &ssm.GetParameterInput{ | ||
| Name: aws.String("/modernising-lpa/s3-batch-configuration/" + environment + "/s3_batch_configuration"), | ||
| }) | ||
| if err != nil { | ||
| return configVars{}, fmt.Errorf("failed to retrieve parameter: %w", err) | ||
| } | ||
|
|
||
| var vars configVars | ||
| if err := json.Unmarshal([]byte(*param.Parameter.Value), &vars); err != nil { | ||
| return configVars{}, fmt.Errorf("failed to unmarshal parameter: %w", err) | ||
| } | ||
|
|
||
| vars.Environment = environment | ||
| return vars, nil | ||
| } | ||
|
|
||
| func createJob(ctx context.Context, cfg aws.Config, vars configVars) (string, error) { | ||
| controlClient := s3control.NewFromConfig(cfg) | ||
| requestToken := uuid.NewString() | ||
|
|
||
| resp, err := controlClient.CreateJob(ctx, &s3control.CreateJobInput{ | ||
| AccountId: aws.String(vars.AccountID), | ||
| ConfirmationRequired: aws.Bool(false), | ||
| Operation: &types.JobOperation{ | ||
| S3ReplicateObject: &types.S3ReplicateObjectOperation{}, | ||
| }, | ||
| Report: &types.JobReport{ | ||
| Enabled: true, | ||
| Bucket: aws.String(vars.ReportAndManifestsBucket), | ||
| Format: types.JobReportFormatReportCsv20180820, | ||
| ReportScope: types.JobReportScopeAllTasks, | ||
| }, | ||
| ClientRequestToken: aws.String(requestToken), | ||
| Description: aws.String("S3 replication " + vars.Environment + " - golang"), | ||
| Priority: aws.Int32(10), | ||
| RoleArn: aws.String(vars.RoleARN), | ||
| ManifestGenerator: &types.JobManifestGeneratorMemberS3JobManifestGenerator{ | ||
| Value: types.S3JobManifestGenerator{ | ||
| EnableManifestOutput: true, | ||
| ExpectedBucketOwner: aws.String(vars.AccountID), | ||
| SourceBucket: aws.String(vars.SourceBucket), | ||
| Filter: &types.JobManifestGeneratorFilter{ | ||
| EligibleForReplication: aws.Bool(true), | ||
| ObjectReplicationStatuses: []types.ReplicationStatus{types.ReplicationStatusFailed, types.ReplicationStatusNone}, | ||
| }, | ||
| ManifestOutputLocation: &types.S3ManifestOutputLocation{ | ||
| ExpectedManifestBucketOwner: aws.String(vars.AccountID), | ||
| Bucket: aws.String(vars.ReportAndManifestsBucket), | ||
| ManifestEncryption: &types.GeneratedManifestEncryption{ | ||
| SSES3: &types.SSES3Encryption{}, | ||
| }, | ||
| ManifestFormat: types.GeneratedManifestFormatS3InventoryReportCsv20211130, | ||
| }, | ||
| }, | ||
| }, | ||
| }) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
|
|
||
| return *resp.JobId, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| FROM golang:1.23.2-alpine AS build | ||
|
|
||
| WORKDIR /app | ||
|
|
||
| COPY --link go.mod go.sum ./ | ||
| RUN go mod download | ||
|
|
||
| COPY --link cmd/create-s3-replication-job ./cmd/create-s3-replication-job | ||
| COPY --link internal ./internal | ||
|
|
||
| RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -o create-s3-replication-job ./cmd/create-s3-replication-job | ||
|
|
||
| FROM public.ecr.aws/lambda/provided:al2023.2024.10.14.12 AS dev | ||
|
|
||
| WORKDIR /app | ||
|
|
||
| COPY --from=build /app/create-s3-replication-job /var/task/create-s3-replication-job | ||
| COPY --link lang ./lang | ||
| COPY --link ./docker/adot-collector/ /opt | ||
| COPY --link docker/aws-lambda-rie ./aws-lambda-rie | ||
|
|
||
| ENV AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler | ||
| ENV OPENTELEMETRY_COLLECTOR_CONFIG_FILE="/opt/config/config.yaml" | ||
|
|
||
| ENTRYPOINT ["./create-s3-replication-job"] | ||
|
|
||
| FROM public.ecr.aws/lambda/provided:al2023.2024.10.14.12 AS production | ||
|
|
||
| WORKDIR /app | ||
| COPY --link docker/install_lambda_insights.sh /app/ | ||
|
|
||
| RUN chmod +x "/app/install_lambda_insights.sh" \ | ||
| && /app/install_lambda_insights.sh "${TARGETPLATFORM}" | ||
|
|
||
| COPY --from=build /app/create-s3-replication-job ./create-s3-replication-job | ||
| COPY --link lang ./lang | ||
| COPY --link ./docker/adot-collector/ /opt | ||
|
|
||
| RUN chmod 755 /opt/config/config.yaml | ||
|
|
||
| ENV AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler | ||
| ENV OPENTELEMETRY_COLLECTOR_CONFIG_FILE="/opt/config/config.yaml" | ||
|
|
||
| ENTRYPOINT ["./create-s3-replication-job"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.