From 10dc1ff2dc2fd27b12bb32985df164bf75e75625 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Wed, 14 Feb 2024 16:55:46 +0000 Subject: [PATCH 01/29] Clean up some untidy and unused code --- internal/ddb/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/ddb/client.go b/internal/ddb/client.go index fe130916..62091c8e 100644 --- a/internal/ddb/client.go +++ b/internal/ddb/client.go @@ -18,7 +18,7 @@ type Client struct { } func (c *Client) PutChanges(ctx context.Context, data any, update shared.Update) error { - changesItem, err := dynamodbattribute.MarshalMap(map[string]interface{}{ + changesItem, _ := dynamodbattribute.MarshalMap(map[string]interface{}{ "uid": update.Uid, "applied": update.Applied, "author": update.Author, @@ -94,7 +94,7 @@ func (c *Client) Get(ctx context.Context, uid string) (shared.Lpa, error) { return lpa, err } -func New(endpoint, tableName string, changesTableName string) *Client { +func New(endpoint, tableName, changesTableName string) *Client { sess := session.Must(session.NewSession()) sess.Config.Endpoint = &endpoint From 04e7dedf3669e4f9a21e868bbcce43ee4d498fa3 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Wed, 14 Feb 2024 16:56:17 +0000 Subject: [PATCH 02/29] Basic interface and constructor for S3 client --- go.mod | 8 +++- go.sum | 34 +++++++---------- internal/objectstore/client.go | 58 +++++++++++++++++++++++++++++ internal/objectstore/client_test.go | 23 ++++++++++++ 4 files changed, 101 insertions(+), 22 deletions(-) create mode 100644 internal/objectstore/client.go create mode 100644 internal/objectstore/client_test.go diff --git a/go.mod b/go.mod index e0271f1d..392b0da0 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.13.2 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.29.0 github.com/aws/aws-sdk-go-v2/service/eventbridge v1.29.3 + github.com/aws/aws-sdk-go-v2/service/s3 v1.50.3 github.com/aws/aws-xray-sdk-go v1.8.3 github.com/golang-jwt/jwt/v5 v5.2.0 github.com/google/go-cmp v0.6.0 @@ -22,6 +23,7 @@ require ( require ( github.com/andybalholm/brotli v1.0.6 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.0 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.0 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.1 // indirect @@ -29,8 +31,10 @@ require ( github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.1 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.19.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.1 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.19.0 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.22.0 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.27.0 // indirect diff --git a/go.sum b/go.sum index 78953a52..4fe1287a 100644 --- a/go.sum +++ b/go.sum @@ -8,26 +8,20 @@ github.com/aws/aws-sdk-go v1.50.20 h1:xfAnSDVf/azIWTVQXQODp89bubvCS85r70O3nuQ4dn github.com/aws/aws-sdk-go v1.50.20/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.25.1 h1:P7hU6A5qEdmajGwvae/zDkOq+ULLC9tQBTwqqiwFGpI= github.com/aws/aws-sdk-go-v2 v1.25.1/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo= -github.com/aws/aws-sdk-go-v2/config v1.26.6 h1:Z/7w9bUqlRI0FFQpetVuFYEsjzE3h7fpU6HuGmfPL/o= -github.com/aws/aws-sdk-go-v2/config v1.26.6/go.mod h1:uKU6cnDmYCvJ+pxO9S4cWDb2yWWIH5hra+32hVh1MI4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo= github.com/aws/aws-sdk-go-v2/config v1.27.0 h1:J5sdGCAHuWKIXLeXiqr8II/adSvetkx0qdZwdbXXpb0= github.com/aws/aws-sdk-go-v2/config v1.27.0/go.mod h1:cfh8v69nuSUohNFMbIISP2fhmblGmYEOKs5V53HiHnk= -github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8= -github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0= github.com/aws/aws-sdk-go-v2/credentials v1.17.0 h1:lMW2x6sKBsiAJrpi1doOXqWFyEPoE886DTb1X0wb7So= github.com/aws/aws-sdk-go-v2/credentials v1.17.0/go.mod h1:uT41FIH8cCIxOdUYIL0PYyHlL1NoneDuDSCwg5VE/5o= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.13.2 h1:ksCAKvVacJbsCJAUWaCk4ZS254NByOKlB8V4dGVWC9c= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.13.2/go.mod h1:vtaNpWHO0v6kWfS27bLuU9dklVj1YmdY/uSc4FqhBE0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.0 h1:xWCwjjvVz2ojYTP4kBKUuUh9ZrXfcAXpflhOUUeXg1k= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.0/go.mod h1:j3fACuqXg4oMTQOR2yY7m0NmJY0yBK4L4sLsRXq1Ins= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.1 h1:evvi7FbTAoFxdP/mixmP7LIYzQWAmzBcwNB/es9XPNc= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.1/go.mod h1:rH61DT6FDdikhPghymripNUCsf+uVF4Cnk4c4DBKH64= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.1 h1:RAnaIrbxPtlXNVI/OIlh1sidTQ3e1qM6LRjs7N0bE0I= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.1/go.mod h1:nbgAGkH5lk0RZRMh6A4K/oG6Xj11eC/1CyDow+DUAFI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 h1:n3GDfwqF2tzEkXlv5cuy4iy7LpKDtqDMcNLfZDu9rls= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.1 h1:rtYJd3w6IWCTVS8vmMaiXjW198noh2PBm5CiXyJea9o= @@ -38,22 +32,22 @@ github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.19.1 h1:Wd1F42HO5ZJ+auc4 github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.19.1/go.mod h1:0FgUg08+1knEoYHo0pa8ogm7D9sjH79lHnRzCNGk/6Q= github.com/aws/aws-sdk-go-v2/service/eventbridge v1.29.3 h1:m/JoWWQI/4Ka9WqTgv9ZupD2zePqVMW8PLmLwr+fiGg= github.com/aws/aws-sdk-go-v2/service/eventbridge v1.29.3/go.mod h1:efCw7VuDRT7Jzj75Tu4Wfx6Pm5Yh6JR2SPSoL7FI1CM= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.0 h1:a33HuFlO0KsveiP90IUJh8Xr/cx9US2PqkSroaLc+o8= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.0/go.mod h1:SxIkWpByiGbhbHYTo9CMTUnx2G4p4ZQMrDPcRRy//1c= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.0 h1:SHN/umDLTmFTmYfI+gkanz6da3vK8Kvj/5wkqnTHbuA= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.0/go.mod h1:l8gPU5RYGOFHJqWEpPMoRTP0VoaWQSkJdKo+hwWnnDA= -github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow= -github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.1 h1:5Wxh862HkXL9CbQ83BIkWKLIgQapGeuh5zG2G9OZtQk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.1/go.mod h1:V7GLA01pNUxMCYSQsibdVrqUrNIYIT/9lCOyR8ExNvQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.1 h1:cVP8mng1RjDyI3JN/AXFCn5FHNlsBaBH0/MBtG1bg0o= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.1/go.mod h1:C8sQjoyAsdfjC7hpy4+S6B92hnFzx0d0UAyHicaOTIE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.1 h1:OYmmIcyw19f7x0qLBLQ3XsrCZSSyLhxd9GXng5evsN4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.1/go.mod h1:s5rqdn74Vdg10k61Pwf4ZHEApOSD6CKRe6qpeHDq32I= +github.com/aws/aws-sdk-go-v2/service/route53 v1.6.2 h1:OsggywXCk9iFKdu2Aopg3e1oJITIuyW36hA/B0rqupE= +github.com/aws/aws-sdk-go-v2/service/route53 v1.6.2/go.mod h1:ZnAMilx42P7DgIrdjlWCkNIGSBLzeyk6T31uB8oGTwY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.50.3 h1:Cv/HH7sLzEdJMYQi4MCNHxZeyubQNOOIdVc0VU0lo3Q= +github.com/aws/aws-sdk-go-v2/service/s3 v1.50.3/go.mod h1:lTW7O4iMAnO2o7H3XJTvqaWFZCH6zIPs+eP7RdG/yp0= github.com/aws/aws-sdk-go-v2/service/sso v1.19.0 h1:u6OkVDxtBPnxPkZ9/63ynEe+8kHbtS5IfaC4PzVxzWM= github.com/aws/aws-sdk-go-v2/service/sso v1.19.0/go.mod h1:YqbU3RS/pkDVu+v+Nwxvn0i1WB0HkNWEePWbmODEbbs= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.22.0 h1:6DL0qu5+315wbsAEEmzK+P9leRwNbkp+lGjPC+CEvb8= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.22.0/go.mod h1:olUAyg+FaoFaL/zFaeQQONjOZ9HXoxgvI/c7mQTYz7M= -github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= -github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= github.com/aws/aws-sdk-go-v2/service/sts v1.27.0 h1:cjTRjh700H36MQ8M0LnDn33W3JmwC77mdxIIyPWCdpM= github.com/aws/aws-sdk-go-v2/service/sts v1.27.0/go.mod h1:nXfOBMWPokIbOY+Gi7a1psWMSvskUCemZzI+SMB7Akc= github.com/aws/aws-xray-sdk-go v1.8.3 h1:S8GdgVncBRhzbNnNUgTPwhEqhwt2alES/9rLASyhxjU= diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go new file mode 100644 index 00000000..aa27f878 --- /dev/null +++ b/internal/objectstore/client.go @@ -0,0 +1,58 @@ +package objectstore + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-xray-sdk-go/instrumentation/awsv2" +) + +type Client interface { + Put(name string, obj any) error + Get(name string) (StoredObject, error) +} + +type StoredObject struct { + name string +} + +type S3Client struct { + awsClient *s3.Client +} + +func (c *S3Client) Put(name string, obj any) error { + /*c.awsClient.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + Body: file, + })*/ + + return nil +} + +func (c *S3Client) Get(name string) (StoredObject, error) { + return StoredObject{}, nil +} + +func New(endpoint string) Client { + cfg, err := config.LoadDefaultConfig(context.Background()) + if err != nil { + panic(err) + } + + // xray instrumentation + awsv2.AWSV2Instrumentor(&cfg.APIOptions) + + awsClient := s3.NewFromConfig( + cfg, + func (o *s3.Options) { + o.BaseEndpoint = aws.String(endpoint) + }, + ) + + return &S3Client{ + awsClient: awsClient, + } +} diff --git a/internal/objectstore/client_test.go b/internal/objectstore/client_test.go new file mode 100644 index 00000000..3296b052 --- /dev/null +++ b/internal/objectstore/client_test.go @@ -0,0 +1,23 @@ +package objectstore + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPutFailureLpaAlreadyExists(t *testing.T) { + c := New("http://localhost:4566") + obj, err := c.Get("M-EEEE-QQQQ-TTYY") + assert.NotEqual(t, nil, obj) + assert.Equal(t, nil, err) +} + +func TestPutSuccess(t *testing.T) { +} + +func TestGetFailureLpaDoesNotExist(t *testing.T) { +} + +func TestGetSuccess(t *testing.T) { +} From 4f65b60b1ba2f7af6780629fae3102378b8026e9 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Thu, 15 Feb 2024 15:54:31 +0000 Subject: [PATCH 03/29] Expose S3 port from localstack in local dev --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 2619722b..b476fa62 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,6 +79,8 @@ services: localstack: image: localstack/localstack:3.1 + ports: + - "4566:4566" volumes: - "./localstack/init:/etc/localstack/init/ready.d" - "./localstack/wait:/scripts/wait" From 874d932b1fe9d9dc157f4c5a22c97c132b3cc277 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Thu, 15 Feb 2024 15:55:06 +0000 Subject: [PATCH 04/29] Add FromJSON() method to populate LPA from S3 JSON --- internal/shared/lpa.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/shared/lpa.go b/internal/shared/lpa.go index e20f3ad9..81273247 100644 --- a/internal/shared/lpa.go +++ b/internal/shared/lpa.go @@ -1,6 +1,10 @@ package shared -import "time" +import ( + "encoding/json" + "io" + "time" +) type LpaInit struct { LpaType LpaType `json:"lpaType"` @@ -47,3 +51,7 @@ const ( LpaStatusProcessing = LpaStatus("processing") LpaStatusRegistered = LpaStatus("registered") ) + +func (l *Lpa) FromJSON(reader io.Reader) error { + return json.NewDecoder(reader).Decode(l) +} From 312673d7daf286b20987705e071d9b44b056bfc6 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Thu, 15 Feb 2024 15:56:34 +0000 Subject: [PATCH 05/29] Working GET and PUT to S3 bucket --- internal/objectstore/client.go | 71 +++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go index aa27f878..f5433fce 100644 --- a/internal/objectstore/client.go +++ b/internal/objectstore/client.go @@ -1,58 +1,75 @@ package objectstore import ( + "bytes" "context" + "encoding/json" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-xray-sdk-go/instrumentation/awsv2" ) type Client interface { - Put(name string, obj any) error - Get(name string) (StoredObject, error) + Put(objectKey string, obj any) (*s3.PutObjectOutput, error) + Get(objectKey string) (*s3.GetObjectOutput, error) } -type StoredObject struct { - name string +type S3Client struct { + bucketName string + awsClient *s3.Client } -type S3Client struct { - awsClient *s3.Client +func (c *S3Client) Put(objectKey string, obj any) (*s3.PutObjectOutput, error) { + b, err := json.Marshal(obj) + if err != nil { + return &s3.PutObjectOutput{}, err + } + + // first return value is output + return c.awsClient.PutObject( + context.Background(), + &s3.PutObjectInput{ + Bucket: aws.String(c.bucketName), + Key: aws.String(objectKey), + Body: bytes.NewReader(b), + }, + ) } -func (c *S3Client) Put(name string, obj any) error { - /*c.awsClient.PutObject(context.TODO(), &s3.PutObjectInput{ - Bucket: aws.String(bucketName), - Key: aws.String(objectKey), - Body: file, - })*/ +func (c *S3Client) Get(objectKey string) (*s3.GetObjectOutput, error) { + return c.awsClient.GetObject( + context.Background(), + &s3.GetObjectInput{ + Bucket: aws.String(c.bucketName), + Key: aws.String(objectKey), + }, + ) +} - return nil +type endpointResolver struct { + URL string } -func (c *S3Client) Get(name string) (StoredObject, error) { - return StoredObject{}, nil +func (er *endpointResolver) ResolveEndpoint(service, region string) (aws.Endpoint, error) { + return aws.Endpoint{ URL: er.URL, HostnameImmutable: true, }, nil } -func New(endpoint string) Client { - cfg, err := config.LoadDefaultConfig(context.Background()) +// set endpoint to "" outside dev to use default resolver +func New(endpointURL string) Client { + cfg, err := config.LoadDefaultConfig( + context.Background(), + config.WithEndpointResolver(&endpointResolver{ URL: endpointURL }), + ) + if err != nil { panic(err) } - // xray instrumentation - awsv2.AWSV2Instrumentor(&cfg.APIOptions) - - awsClient := s3.NewFromConfig( - cfg, - func (o *s3.Options) { - o.BaseEndpoint = aws.String(endpoint) - }, - ) + awsClient := s3.NewFromConfig(cfg) return &S3Client{ + bucketName: "opg-lpa-store-static-eu-west-1", awsClient: awsClient, } } From b537a2231fe6fd92e7b2e4ef368e017cafc4d23b Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Thu, 15 Feb 2024 17:29:04 +0000 Subject: [PATCH 06/29] Incorporate s3 client into create lambda --- docker-compose.yml | 4 ++- internal/objectstore/client.go | 10 ++----- lambda/create/main.go | 50 +++++++++++++++++++++++++++++----- 3 files changed, 48 insertions(+), 16 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b476fa62..b976defc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,12 +12,14 @@ services: - DIR=create environment: AWS_REGION: eu-west-1 - AWS_ACCESS_KEY_ID: localstack AWS_DYNAMODB_ENDPOINT: http://localstack:4566 AWS_EVENTBRIDGE_ENDPOINT: http://localstack:4566 + AWS_S3_ENDPOINT: http://localstack:4566 + AWS_ACCESS_KEY_ID: localstack AWS_SECRET_ACCESS_KEY: localstack DDB_TABLE_NAME_DEEDS: deeds DDB_TABLE_NAME_CHANGES: changes + S3_BUCKET_NAME_ORIGINAL: opg-lpa-store-static-eu-west-1 EVENT_BUS_NAME: local-main JWT_SECRET_KEY: ${JWT_SECRET_KEY:-secret} volumes: diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go index f5433fce..15df0c10 100644 --- a/internal/objectstore/client.go +++ b/internal/objectstore/client.go @@ -10,11 +10,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" ) -type Client interface { - Put(objectKey string, obj any) (*s3.PutObjectOutput, error) - Get(objectKey string) (*s3.GetObjectOutput, error) -} - type S3Client struct { bucketName string awsClient *s3.Client @@ -26,7 +21,6 @@ func (c *S3Client) Put(objectKey string, obj any) (*s3.PutObjectOutput, error) { return &s3.PutObjectOutput{}, err } - // first return value is output return c.awsClient.PutObject( context.Background(), &s3.PutObjectInput{ @@ -56,7 +50,7 @@ func (er *endpointResolver) ResolveEndpoint(service, region string) (aws.Endpoin } // set endpoint to "" outside dev to use default resolver -func New(endpointURL string) Client { +func NewS3Client(bucketName, endpointURL string) *S3Client { cfg, err := config.LoadDefaultConfig( context.Background(), config.WithEndpointResolver(&endpointResolver{ URL: endpointURL }), @@ -69,7 +63,7 @@ func New(endpointURL string) Client { awsClient := s3.NewFromConfig(cfg) return &S3Client{ - bucketName: "opg-lpa-store-static-eu-west-1", + bucketName: bucketName, awsClient: awsClient, } } diff --git a/lambda/create/main.go b/lambda/create/main.go index fc9d49a2..e589930f 100644 --- a/lambda/create/main.go +++ b/lambda/create/main.go @@ -3,14 +3,19 @@ package main import ( "context" "encoding/json" + "errors" + "fmt" "os" "time" "github.com/aws/aws-lambda-go/events" "github.com/aws/aws-lambda-go/lambda" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/ministryofjustice/opg-data-lpa-store/internal/ddb" "github.com/ministryofjustice/opg-data-lpa-store/internal/event" + "github.com/ministryofjustice/opg-data-lpa-store/internal/objectstore" "github.com/ministryofjustice/opg-data-lpa-store/internal/shared" "github.com/ministryofjustice/opg-go-common/logging" ) @@ -28,15 +33,21 @@ type Store interface { Get(ctx context.Context, uid string) (shared.Lpa, error) } +type S3Client interface { + Put(objectKey string, obj any) (*s3.PutObjectOutput, error) + Get(objectKey string) (*s3.GetObjectOutput, error) +} + type Verifier interface { VerifyHeader(events.APIGatewayProxyRequest) (*shared.LpaStoreClaims, error) } type Lambda struct { - eventClient EventClient - store Store - verifier Verifier - logger Logger + eventClient EventClient + s3client S3Client + store Store + verifier Verifier + logger Logger } func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) { @@ -77,10 +88,10 @@ func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequ } // validation - errors := Validate(input) - if len(errors) > 0 { + errs := Validate(input) + if len(errs) > 0 { problem := shared.ProblemInvalidRequest - problem.Errors = errors + problem.Errors = errs return problem.Respond() } @@ -92,6 +103,27 @@ func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequ // save err = l.store.Put(ctx, data) + if err != nil { + l.logger.Print(err) + return shared.ProblemInternalServerError.Respond() + } + + // save a copy of the original to permanent storage, + // but only if the key doesn't already exist + objectKey := fmt.Sprintf("%s/donor-executed-lpa.json", data.Uid) + _, err = l.s3client.Get(objectKey) + if err == nil { + // 200 => bad (object already exists) + err = errors.New(fmt.Sprintf("Could not save donor executed LPA as key %s already exists", objectKey)) + l.logger.Print(err) + return shared.ProblemInvalidRequest.Respond() + } + + // 404 => good (object should not already exist) + var nsk *types.NoSuchKey + if errors.As(err, &nsk) { + _, err = l.s3client.Put(objectKey, data) + } if err != nil { l.logger.Print(err) @@ -130,6 +162,10 @@ func main() { os.Getenv("DDB_TABLE_NAME_DEEDS"), os.Getenv("DDB_TABLE_NAME_CHANGES"), ), + s3client: objectstore.NewS3Client( + os.Getenv("S3_BUCKET_NAME_ORIGINAL"), + os.Getenv("AWS_S3_ENDPOINT"), + ), verifier: shared.NewJWTVerifier(), logger: logger, } From 709d5b30a2baa41b8d8a9e07b97d06ec815bdc54 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Mon, 19 Feb 2024 17:03:24 +0000 Subject: [PATCH 07/29] Update dependencies --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 4fe1287a..ab0d0161 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,6 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.1 h1:cVP8mng1R github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.1/go.mod h1:C8sQjoyAsdfjC7hpy4+S6B92hnFzx0d0UAyHicaOTIE= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.1 h1:OYmmIcyw19f7x0qLBLQ3XsrCZSSyLhxd9GXng5evsN4= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.1/go.mod h1:s5rqdn74Vdg10k61Pwf4ZHEApOSD6CKRe6qpeHDq32I= -github.com/aws/aws-sdk-go-v2/service/route53 v1.6.2 h1:OsggywXCk9iFKdu2Aopg3e1oJITIuyW36hA/B0rqupE= -github.com/aws/aws-sdk-go-v2/service/route53 v1.6.2/go.mod h1:ZnAMilx42P7DgIrdjlWCkNIGSBLzeyk6T31uB8oGTwY= github.com/aws/aws-sdk-go-v2/service/s3 v1.50.3 h1:Cv/HH7sLzEdJMYQi4MCNHxZeyubQNOOIdVc0VU0lo3Q= github.com/aws/aws-sdk-go-v2/service/s3 v1.50.3/go.mod h1:lTW7O4iMAnO2o7H3XJTvqaWFZCH6zIPs+eP7RdG/yp0= github.com/aws/aws-sdk-go-v2/service/sso v1.19.0 h1:u6OkVDxtBPnxPkZ9/63ynEe+8kHbtS5IfaC4PzVxzWM= From 28aae1843eb4899bfdf4392af87c9a9c9c22b766 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Mon, 19 Feb 2024 17:03:36 +0000 Subject: [PATCH 08/29] Unit test for S3 client --- internal/objectstore/client.go | 7 ++++- internal/objectstore/client_test.go | 47 ++++++++++++++++++++++++----- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go index 15df0c10..61a99892 100644 --- a/internal/objectstore/client.go +++ b/internal/objectstore/client.go @@ -10,9 +10,14 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" ) +type AwsS3Client interface { + PutObject(ctx context.Context, input *s3.PutObjectInput, opts ...func(*s3.Options)) (*s3.PutObjectOutput, error) + GetObject(ctx context.Context, input *s3.GetObjectInput, opts ...func(*s3.Options)) (*s3.GetObjectOutput, error) +} + type S3Client struct { bucketName string - awsClient *s3.Client + awsClient AwsS3Client } func (c *S3Client) Put(objectKey string, obj any) (*s3.PutObjectOutput, error) { diff --git a/internal/objectstore/client_test.go b/internal/objectstore/client_test.go index 3296b052..60cdd32a 100644 --- a/internal/objectstore/client_test.go +++ b/internal/objectstore/client_test.go @@ -1,23 +1,54 @@ package objectstore import ( + "context" "testing" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) -func TestPutFailureLpaAlreadyExists(t *testing.T) { - c := New("http://localhost:4566") - obj, err := c.Get("M-EEEE-QQQQ-TTYY") - assert.NotEqual(t, nil, obj) - assert.Equal(t, nil, err) +type mockAwsClient struct { + mock.Mock } -func TestPutSuccess(t *testing.T) { +func (m *mockAwsClient) PutObject(ctx context.Context, input *s3.PutObjectInput, opts ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + args := m.Called(ctx, input) + return args.Get(0).(*s3.PutObjectOutput), args.Error(1) } -func TestGetFailureLpaDoesNotExist(t *testing.T) { +func (m *mockAwsClient) GetObject(ctx context.Context, input *s3.GetObjectInput, opts ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + args := m.Called(ctx, input) + return args.Get(0).(*s3.GetObjectOutput), args.Error(1) } -func TestGetSuccess(t *testing.T) { +func TestPut(t *testing.T) { + client := mockAwsClient{} + client.On("PutObject", mock.Anything, mock.Anything).Return(&s3.PutObjectOutput{}, nil) + + c := S3Client{ + bucketName: "bucket1", + awsClient: &client, + } + + _, err := c.Put("anobject", struct{ID int}{ID: 1}) + + assert.Equal(t, nil, err) + client.AssertExpectations(t) +} + +func TestGet(t *testing.T) { + client := mockAwsClient{} + client.On("GetObject", mock.Anything, mock.Anything).Return(&s3.GetObjectOutput{}, nil) + + c := S3Client{ + bucketName: "bucket1", + awsClient: &client, + } + + _, err := c.Get("anotherobject") + + assert.Equal(t, nil, err) + client.AssertExpectations(t) } From f0f8dd6ca9c01fb9c90f340cdfd0a75717682fcd Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Mon, 19 Feb 2024 17:25:49 +0000 Subject: [PATCH 09/29] Correct code style --- lambda/create/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/create/main.go b/lambda/create/main.go index e589930f..a26d5c18 100644 --- a/lambda/create/main.go +++ b/lambda/create/main.go @@ -114,7 +114,7 @@ func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequ _, err = l.s3client.Get(objectKey) if err == nil { // 200 => bad (object already exists) - err = errors.New(fmt.Sprintf("Could not save donor executed LPA as key %s already exists", objectKey)) + err = fmt.Errorf("Could not save donor executed LPA as key %s already exists", objectKey) l.logger.Print(err) return shared.ProblemInvalidRequest.Respond() } From 3d56b45b9a20410f86a8e1ea184127e61b8ae55a Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Mon, 19 Feb 2024 17:51:19 +0000 Subject: [PATCH 10/29] Refactor S3 client to make it unit-testable --- internal/objectstore/static_lpa_storage.go | 42 +++++++++++++++++++ lambda/create/main.go | 47 ++++++++-------------- 2 files changed, 58 insertions(+), 31 deletions(-) create mode 100644 internal/objectstore/static_lpa_storage.go diff --git a/internal/objectstore/static_lpa_storage.go b/internal/objectstore/static_lpa_storage.go new file mode 100644 index 00000000..5add5955 --- /dev/null +++ b/internal/objectstore/static_lpa_storage.go @@ -0,0 +1,42 @@ +package objectstore + +import ( + "errors" + "fmt" + + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/ministryofjustice/opg-data-lpa-store/internal/shared" +) + +/** + * For saving static copy of original LPA to S3 + */ +type StaticLpaStorage struct { + client *S3Client +} + +func (sls *StaticLpaStorage) Save(lpa *shared.Lpa) error { + // save a copy of the original to permanent storage, + // but only if the key doesn't already exist + objectKey := fmt.Sprintf("%s/donor-executed-lpa.json", lpa.Uid) + _, err := sls.client.Get(objectKey) + + if err == nil { + // no error and 200 => bad (object already exists) + return fmt.Errorf("Could not save donor executed LPA as key %s already exists", objectKey) + } + + // error which is a 404 => good (object should not already exist) + var nsk *types.NoSuchKey + if errors.As(err, &nsk) { + _, err = sls.client.Put(objectKey, lpa) + } + + return err +} + +func NewStaticLpaStorage(client *S3Client) *StaticLpaStorage { + return &StaticLpaStorage{ + client: client, + } +} diff --git a/lambda/create/main.go b/lambda/create/main.go index a26d5c18..d3e2f926 100644 --- a/lambda/create/main.go +++ b/lambda/create/main.go @@ -3,8 +3,6 @@ package main import ( "context" "encoding/json" - "errors" - "fmt" "os" "time" @@ -33,9 +31,8 @@ type Store interface { Get(ctx context.Context, uid string) (shared.Lpa, error) } -type S3Client interface { - Put(objectKey string, obj any) (*s3.PutObjectOutput, error) - Get(objectKey string) (*s3.GetObjectOutput, error) +type StaticLpaStorage interface { + Save(lpa *shared.Lpa) error } type Verifier interface { @@ -43,11 +40,11 @@ type Verifier interface { } type Lambda struct { - eventClient EventClient - s3client S3Client - store Store - verifier Verifier - logger Logger + eventClient EventClient + staticLpaStorage StaticLpaStorage + store Store + verifier Verifier + logger Logger } func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) { @@ -108,22 +105,8 @@ func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequ return shared.ProblemInternalServerError.Respond() } - // save a copy of the original to permanent storage, - // but only if the key doesn't already exist - objectKey := fmt.Sprintf("%s/donor-executed-lpa.json", data.Uid) - _, err = l.s3client.Get(objectKey) - if err == nil { - // 200 => bad (object already exists) - err = fmt.Errorf("Could not save donor executed LPA as key %s already exists", objectKey) - l.logger.Print(err) - return shared.ProblemInvalidRequest.Respond() - } - - // 404 => good (object should not already exist) - var nsk *types.NoSuchKey - if errors.As(err, &nsk) { - _, err = l.s3client.Put(objectKey, data) - } + // save to static storage as JSON + err = l.staticLpaStorage.Save(&data) if err != nil { l.logger.Print(err) @@ -157,17 +140,19 @@ func main() { l := &Lambda{ eventClient: event.NewClient(awsConfig, os.Getenv("EVENT_BUS_NAME")), - store: ddb.New( + store: ddb.New( os.Getenv("AWS_DYNAMODB_ENDPOINT"), os.Getenv("DDB_TABLE_NAME_DEEDS"), os.Getenv("DDB_TABLE_NAME_CHANGES"), ), - s3client: objectstore.NewS3Client( - os.Getenv("S3_BUCKET_NAME_ORIGINAL"), - os.Getenv("AWS_S3_ENDPOINT"), + staticLpaStorage: objectstore.NewStaticLpaStorage( + objectstore.NewS3Client( + os.Getenv("S3_BUCKET_NAME_ORIGINAL"), + os.Getenv("AWS_S3_ENDPOINT"), + ), ), verifier: shared.NewJWTVerifier(), - logger: logger, + logger: logger, } lambda.Start(l.HandleEvent) From d1d49037497671484b69c08e371fd306b7f508ff Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Tue, 20 Feb 2024 11:48:35 +0000 Subject: [PATCH 11/29] Add unit test for LPA storage client --- internal/objectstore/client.go | 4 +- internal/objectstore/static_lpa_storage.go | 10 +- .../objectstore/static_lpa_storage_test.go | 91 +++++++++++++++++++ 3 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 internal/objectstore/static_lpa_storage_test.go diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go index 61a99892..4f9572b6 100644 --- a/internal/objectstore/client.go +++ b/internal/objectstore/client.go @@ -10,14 +10,14 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" ) -type AwsS3Client interface { +type awsS3Client interface { PutObject(ctx context.Context, input *s3.PutObjectInput, opts ...func(*s3.Options)) (*s3.PutObjectOutput, error) GetObject(ctx context.Context, input *s3.GetObjectInput, opts ...func(*s3.Options)) (*s3.GetObjectOutput, error) } type S3Client struct { bucketName string - awsClient AwsS3Client + awsClient awsS3Client } func (c *S3Client) Put(objectKey string, obj any) (*s3.PutObjectOutput, error) { diff --git a/internal/objectstore/static_lpa_storage.go b/internal/objectstore/static_lpa_storage.go index 5add5955..dbc5f117 100644 --- a/internal/objectstore/static_lpa_storage.go +++ b/internal/objectstore/static_lpa_storage.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/ministryofjustice/opg-data-lpa-store/internal/shared" ) @@ -11,8 +12,13 @@ import ( /** * For saving static copy of original LPA to S3 */ +type s3Client interface { + Put(objectKey string, obj any) (*s3.PutObjectOutput, error) + Get(objectKey string) (*s3.GetObjectOutput, error) +} + type StaticLpaStorage struct { - client *S3Client + client s3Client } func (sls *StaticLpaStorage) Save(lpa *shared.Lpa) error { @@ -35,7 +41,7 @@ func (sls *StaticLpaStorage) Save(lpa *shared.Lpa) error { return err } -func NewStaticLpaStorage(client *S3Client) *StaticLpaStorage { +func NewStaticLpaStorage(client s3Client) *StaticLpaStorage { return &StaticLpaStorage{ client: client, } diff --git a/internal/objectstore/static_lpa_storage_test.go b/internal/objectstore/static_lpa_storage_test.go new file mode 100644 index 00000000..fa54ae00 --- /dev/null +++ b/internal/objectstore/static_lpa_storage_test.go @@ -0,0 +1,91 @@ +package objectstore + +import ( + "errors" + "fmt" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/ministryofjustice/opg-data-lpa-store/internal/shared" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +type testCase struct { + description string + mockClientGetError error + expectPut bool + mockClientPutError error + expectedReturnError error +} + +type mockS3Client struct { + mock.Mock +} + +func (m *mockS3Client) Put(objectKey string, obj any) (*s3.PutObjectOutput, error) { + args := m.Called(objectKey, obj) + return args.Get(0).(*s3.PutObjectOutput), args.Error(1) +} + +func (m *mockS3Client) Get(objectKey string) (*s3.GetObjectOutput, error) { + args := m.Called(objectKey) + return args.Get(0).(*s3.GetObjectOutput), args.Error(1) +} + +func TestSave(t *testing.T) { + lpa := shared.Lpa{ Uid: "M-QQQQ-EEEE-TTTT" } + expectedObjectKey := fmt.Sprintf("%s/donor-executed-lpa.json", lpa.Uid) + + testCases := []testCase{ + testCase{ + description: "FAIL - object already exists in S3 and shouldn't be overwritten", + mockClientGetError: nil, + expectPut: false, + mockClientPutError: nil, + expectedReturnError: fmt.Errorf("Could not save donor executed LPA as key %s already exists", expectedObjectKey), + }, + + testCase{ + description: "FAIL - S3 get of object key fails due to HTTP error", + mockClientGetError: errors.New("HTTP transport error"), + expectPut: false, + mockClientPutError: nil, + expectedReturnError: errors.New("HTTP transport error"), + }, + + testCase{ + description: "FAIL - S3 put of object key fails due to HTTP error", + mockClientGetError: &types.NoSuchKey{}, + expectPut: true, + mockClientPutError: errors.New("HTTP transport error"), + expectedReturnError: errors.New("HTTP transport error"), + }, + + testCase{ + description: "SUCCESS - object does not exist in S3, and put of object to S3 is OK", + mockClientGetError: &types.NoSuchKey{}, + expectPut: true, + mockClientPutError: nil, + expectedReturnError: nil, + }, + } + + for _, tc := range(testCases) { + t.Run(tc.description, func (t *testing.T) { + mockClient := mockS3Client{} + mockClient.On("Get", expectedObjectKey).Return(&s3.GetObjectOutput{}, tc.mockClientGetError) + + if tc.expectPut { + mockClient.On("Put", expectedObjectKey, mock.Anything).Return(&s3.PutObjectOutput{}, tc.mockClientPutError) + } + + sut := NewStaticLpaStorage(&mockClient) + actualErr := sut.Save(&lpa) + + assert.Equal(t, tc.expectedReturnError, actualErr) + mockClient.AssertExpectations(t) + }) + } +} From d4b788f5f8edd16179b0093cc241411b3d10bb6c Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Thu, 22 Feb 2024 15:45:07 +0000 Subject: [PATCH 12/29] Fix merge error --- lambda/create/main.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/lambda/create/main.go b/lambda/create/main.go index d3e2f926..214230f1 100644 --- a/lambda/create/main.go +++ b/lambda/create/main.go @@ -9,8 +9,6 @@ import ( "github.com/aws/aws-lambda-go/events" "github.com/aws/aws-lambda-go/lambda" "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/ministryofjustice/opg-data-lpa-store/internal/ddb" "github.com/ministryofjustice/opg-data-lpa-store/internal/event" "github.com/ministryofjustice/opg-data-lpa-store/internal/objectstore" From 8936ef63247eecb1ebc2cb6754c3d5693b778fd7 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Thu, 22 Feb 2024 16:43:26 +0000 Subject: [PATCH 13/29] Point app at the static LPA S3 bucket --- terraform/environment/region/lambda.tf | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/terraform/environment/region/lambda.tf b/terraform/environment/region/lambda.tf index b50b7867..baee764d 100644 --- a/terraform/environment/region/lambda.tf +++ b/terraform/environment/region/lambda.tf @@ -17,10 +17,11 @@ module "lambda" { cloudwatch_kms_key_id = aws_kms_key.cloudwatch.arn environment_variables = { - DDB_TABLE_NAME_DEEDS = var.dynamodb_name - DDB_TABLE_NAME_CHANGES = var.dynamodb_name_changes - EVENT_BUS_NAME = var.event_bus.name - JWT_SECRET_KEY = "secret" + DDB_TABLE_NAME_DEEDS = var.dynamodb_name + DDB_TABLE_NAME_CHANGES = var.dynamodb_name_changes + EVENT_BUS_NAME = var.event_bus.name + S3_BUCKET_NAME_ORIGINAL = var.lpa_store_static_bucket.bucket + JWT_SECRET_KEY = "secret" } providers = { From 5adfc1b4addb836df4f9b1e6b7ee8069c86b7e4e Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Mon, 26 Feb 2024 13:01:23 +0000 Subject: [PATCH 14/29] Modify S3 endpoint resolution based on env var --- internal/objectstore/client.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go index 4f9572b6..733cb9ea 100644 --- a/internal/objectstore/client.go +++ b/internal/objectstore/client.go @@ -56,16 +56,20 @@ func (er *endpointResolver) ResolveEndpoint(service, region string) (aws.Endpoin // set endpoint to "" outside dev to use default resolver func NewS3Client(bucketName, endpointURL string) *S3Client { - cfg, err := config.LoadDefaultConfig( - context.Background(), - config.WithEndpointResolver(&endpointResolver{ URL: endpointURL }), - ) + cfg, err := config.LoadDefaultConfig(context.Background()) if err != nil { panic(err) } - awsClient := s3.NewFromConfig(cfg) + var awsClient *s3.Client + if endpointURL != "" { + awsClient = s3.NewFromConfig(cfg, func (o *s3.Options) { + o.BaseEndpoint = aws.String(endpointURL) + }) + } else { + awsClient = s3.NewFromConfig(cfg) + } return &S3Client{ bucketName: bucketName, From a8b6a49ff083f30808180b0c8e3fe2e48fac9811 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Mon, 26 Feb 2024 14:49:35 +0000 Subject: [PATCH 15/29] Add GetObject S3 permission --- terraform/environment/region/iam.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/environment/region/iam.tf b/terraform/environment/region/iam.tf index e1b56ecc..2391785e 100644 --- a/terraform/environment/region/iam.tf +++ b/terraform/environment/region/iam.tf @@ -41,6 +41,7 @@ data "aws_iam_policy_document" "lambda_s3_policy" { ] actions = [ "s3:PutObject", + "s3:GetObject", ] } statement { From 35ba81e7b18768b76adf390e9761c8e993ea6509 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Mon, 26 Feb 2024 15:19:41 +0000 Subject: [PATCH 16/29] Name might be wrong, guessing --- terraform/environment/region/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environment/region/iam.tf b/terraform/environment/region/iam.tf index 2391785e..21de9215 100644 --- a/terraform/environment/region/iam.tf +++ b/terraform/environment/region/iam.tf @@ -23,7 +23,7 @@ data "aws_iam_policy_document" "lambda_dynamodb_policy" { } } -resource "aws_iam_role_policy" "lambda_s3" { +resource "aws_iam_role_policy" "lambda_s3_policy" { for_each = local.functions name = "LambdaAllowS3" role = module.lambda[each.key].iam_role.id From 8f13562f2eac5446bf0858745eafd991385e96bc Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Tue, 27 Feb 2024 09:54:25 +0000 Subject: [PATCH 17/29] Change endpoint resolver to work in CI --- internal/objectstore/client.go | 40 ++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go index 733cb9ea..d929f563 100644 --- a/internal/objectstore/client.go +++ b/internal/objectstore/client.go @@ -4,10 +4,13 @@ import ( "bytes" "context" "encoding/json" + "net/url" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + smithyendpoints "github.com/aws/smithy-go/endpoints" ) type awsS3Client interface { @@ -29,9 +32,10 @@ func (c *S3Client) Put(objectKey string, obj any) (*s3.PutObjectOutput, error) { return c.awsClient.PutObject( context.Background(), &s3.PutObjectInput{ - Bucket: aws.String(c.bucketName), - Key: aws.String(objectKey), - Body: bytes.NewReader(b), + Bucket: aws.String(c.bucketName), + Key: aws.String(objectKey), + Body: bytes.NewReader(b), + ServerSideEncryption: types.ServerSideEncryptionAwsKms, }, ) } @@ -46,30 +50,34 @@ func (c *S3Client) Get(objectKey string) (*s3.GetObjectOutput, error) { ) } -type endpointResolver struct { - URL string +type resolverV2 struct { + URL string } -func (er *endpointResolver) ResolveEndpoint(service, region string) (aws.Endpoint, error) { - return aws.Endpoint{ URL: er.URL, HostnameImmutable: true, }, nil +func (r *resolverV2) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (smithyendpoints.Endpoint, error) { + if r.URL != "" { + u, err := url.Parse(r.URL) + if err != nil { + return smithyendpoints.Endpoint{}, err + } + return smithyendpoints.Endpoint{ URI: *u }, nil + } + + return s3.NewDefaultEndpointResolverV2().ResolveEndpoint(ctx, params) } // set endpoint to "" outside dev to use default resolver func NewS3Client(bucketName, endpointURL string) *S3Client { cfg, err := config.LoadDefaultConfig(context.Background()) - if err != nil { panic(err) } - var awsClient *s3.Client - if endpointURL != "" { - awsClient = s3.NewFromConfig(cfg, func (o *s3.Options) { - o.BaseEndpoint = aws.String(endpointURL) - }) - } else { - awsClient = s3.NewFromConfig(cfg) - } + awsClient := s3.NewFromConfig(cfg, func (o *s3.Options) { + o.EndpointResolverV2 = &resolverV2{ + URL: endpointURL, + } + }) return &S3Client{ bucketName: bucketName, From ba24822b10bb5b9279153712177f100af49c3d6c Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Wed, 28 Feb 2024 08:15:41 +0000 Subject: [PATCH 18/29] PUT static LPA without checking if it exists The bucket is versioned, so if we accidentally write the LPA multiple times, the first version is always the one we want anyway. --- internal/objectstore/static_lpa_storage.go | 48 ---------- .../objectstore/static_lpa_storage_test.go | 91 ------------------- lambda/create/main.go | 12 +-- terraform/environment/region/iam.tf | 3 +- 4 files changed, 7 insertions(+), 147 deletions(-) delete mode 100644 internal/objectstore/static_lpa_storage.go delete mode 100644 internal/objectstore/static_lpa_storage_test.go diff --git a/internal/objectstore/static_lpa_storage.go b/internal/objectstore/static_lpa_storage.go deleted file mode 100644 index dbc5f117..00000000 --- a/internal/objectstore/static_lpa_storage.go +++ /dev/null @@ -1,48 +0,0 @@ -package objectstore - -import ( - "errors" - "fmt" - - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/ministryofjustice/opg-data-lpa-store/internal/shared" -) - -/** - * For saving static copy of original LPA to S3 - */ -type s3Client interface { - Put(objectKey string, obj any) (*s3.PutObjectOutput, error) - Get(objectKey string) (*s3.GetObjectOutput, error) -} - -type StaticLpaStorage struct { - client s3Client -} - -func (sls *StaticLpaStorage) Save(lpa *shared.Lpa) error { - // save a copy of the original to permanent storage, - // but only if the key doesn't already exist - objectKey := fmt.Sprintf("%s/donor-executed-lpa.json", lpa.Uid) - _, err := sls.client.Get(objectKey) - - if err == nil { - // no error and 200 => bad (object already exists) - return fmt.Errorf("Could not save donor executed LPA as key %s already exists", objectKey) - } - - // error which is a 404 => good (object should not already exist) - var nsk *types.NoSuchKey - if errors.As(err, &nsk) { - _, err = sls.client.Put(objectKey, lpa) - } - - return err -} - -func NewStaticLpaStorage(client s3Client) *StaticLpaStorage { - return &StaticLpaStorage{ - client: client, - } -} diff --git a/internal/objectstore/static_lpa_storage_test.go b/internal/objectstore/static_lpa_storage_test.go deleted file mode 100644 index fa54ae00..00000000 --- a/internal/objectstore/static_lpa_storage_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package objectstore - -import ( - "errors" - "fmt" - "testing" - - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/ministryofjustice/opg-data-lpa-store/internal/shared" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -type testCase struct { - description string - mockClientGetError error - expectPut bool - mockClientPutError error - expectedReturnError error -} - -type mockS3Client struct { - mock.Mock -} - -func (m *mockS3Client) Put(objectKey string, obj any) (*s3.PutObjectOutput, error) { - args := m.Called(objectKey, obj) - return args.Get(0).(*s3.PutObjectOutput), args.Error(1) -} - -func (m *mockS3Client) Get(objectKey string) (*s3.GetObjectOutput, error) { - args := m.Called(objectKey) - return args.Get(0).(*s3.GetObjectOutput), args.Error(1) -} - -func TestSave(t *testing.T) { - lpa := shared.Lpa{ Uid: "M-QQQQ-EEEE-TTTT" } - expectedObjectKey := fmt.Sprintf("%s/donor-executed-lpa.json", lpa.Uid) - - testCases := []testCase{ - testCase{ - description: "FAIL - object already exists in S3 and shouldn't be overwritten", - mockClientGetError: nil, - expectPut: false, - mockClientPutError: nil, - expectedReturnError: fmt.Errorf("Could not save donor executed LPA as key %s already exists", expectedObjectKey), - }, - - testCase{ - description: "FAIL - S3 get of object key fails due to HTTP error", - mockClientGetError: errors.New("HTTP transport error"), - expectPut: false, - mockClientPutError: nil, - expectedReturnError: errors.New("HTTP transport error"), - }, - - testCase{ - description: "FAIL - S3 put of object key fails due to HTTP error", - mockClientGetError: &types.NoSuchKey{}, - expectPut: true, - mockClientPutError: errors.New("HTTP transport error"), - expectedReturnError: errors.New("HTTP transport error"), - }, - - testCase{ - description: "SUCCESS - object does not exist in S3, and put of object to S3 is OK", - mockClientGetError: &types.NoSuchKey{}, - expectPut: true, - mockClientPutError: nil, - expectedReturnError: nil, - }, - } - - for _, tc := range(testCases) { - t.Run(tc.description, func (t *testing.T) { - mockClient := mockS3Client{} - mockClient.On("Get", expectedObjectKey).Return(&s3.GetObjectOutput{}, tc.mockClientGetError) - - if tc.expectPut { - mockClient.On("Put", expectedObjectKey, mock.Anything).Return(&s3.PutObjectOutput{}, tc.mockClientPutError) - } - - sut := NewStaticLpaStorage(&mockClient) - actualErr := sut.Save(&lpa) - - assert.Equal(t, tc.expectedReturnError, actualErr) - mockClient.AssertExpectations(t) - }) - } -} diff --git a/lambda/create/main.go b/lambda/create/main.go index 214230f1..8ce5fc9f 100644 --- a/lambda/create/main.go +++ b/lambda/create/main.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "fmt" "os" "time" @@ -104,7 +105,8 @@ func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequ } // save to static storage as JSON - err = l.staticLpaStorage.Save(&data) + objectKey := fmt.Sprintf("%s/donor-executed-lpa.json", lpa.Uid) + _, err := l.staticLpaStorage.Put(objectKey, data) if err != nil { l.logger.Print(err) @@ -143,11 +145,9 @@ func main() { os.Getenv("DDB_TABLE_NAME_DEEDS"), os.Getenv("DDB_TABLE_NAME_CHANGES"), ), - staticLpaStorage: objectstore.NewStaticLpaStorage( - objectstore.NewS3Client( - os.Getenv("S3_BUCKET_NAME_ORIGINAL"), - os.Getenv("AWS_S3_ENDPOINT"), - ), + staticLpaStorage: objectstore.NewS3Client( + os.Getenv("S3_BUCKET_NAME_ORIGINAL"), + os.Getenv("AWS_S3_ENDPOINT"), ), verifier: shared.NewJWTVerifier(), logger: logger, diff --git a/terraform/environment/region/iam.tf b/terraform/environment/region/iam.tf index 21de9215..b0a707b3 100644 --- a/terraform/environment/region/iam.tf +++ b/terraform/environment/region/iam.tf @@ -40,8 +40,7 @@ data "aws_iam_policy_document" "lambda_s3_policy" { "${var.lpa_store_static_bucket.arn}/*", ] actions = [ - "s3:PutObject", - "s3:GetObject", + "s3:PutObject" ] } statement { From d829cf3e6050e422c18931cf8b1cb62b3a65723c Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Wed, 28 Feb 2024 08:29:15 +0000 Subject: [PATCH 19/29] Static analysis fixes --- lambda/create/main.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lambda/create/main.go b/lambda/create/main.go index 8ce5fc9f..7367e444 100644 --- a/lambda/create/main.go +++ b/lambda/create/main.go @@ -10,6 +10,7 @@ import ( "github.com/aws/aws-lambda-go/events" "github.com/aws/aws-lambda-go/lambda" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/ministryofjustice/opg-data-lpa-store/internal/ddb" "github.com/ministryofjustice/opg-data-lpa-store/internal/event" "github.com/ministryofjustice/opg-data-lpa-store/internal/objectstore" @@ -30,8 +31,8 @@ type Store interface { Get(ctx context.Context, uid string) (shared.Lpa, error) } -type StaticLpaStorage interface { - Save(lpa *shared.Lpa) error +type S3Client interface { + Put(objectKey string, obj any) (*s3.PutObjectOutput, error) } type Verifier interface { @@ -40,7 +41,7 @@ type Verifier interface { type Lambda struct { eventClient EventClient - staticLpaStorage StaticLpaStorage + staticLpaStorage S3Client store Store verifier Verifier logger Logger @@ -105,8 +106,8 @@ func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequ } // save to static storage as JSON - objectKey := fmt.Sprintf("%s/donor-executed-lpa.json", lpa.Uid) - _, err := l.staticLpaStorage.Put(objectKey, data) + objectKey := fmt.Sprintf("%s/donor-executed-lpa.json", data.Uid) + _, err = l.staticLpaStorage.Put(objectKey, data) if err != nil { l.logger.Print(err) From 195c1dd05aa9960417a0fd0df2de3375c92df9f7 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Wed, 28 Feb 2024 08:31:35 +0000 Subject: [PATCH 20/29] tabs --- internal/objectstore/client.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go index d929f563..c21d7879 100644 --- a/internal/objectstore/client.go +++ b/internal/objectstore/client.go @@ -24,10 +24,10 @@ type S3Client struct { } func (c *S3Client) Put(objectKey string, obj any) (*s3.PutObjectOutput, error) { - b, err := json.Marshal(obj) - if err != nil { - return &s3.PutObjectOutput{}, err - } + b, err := json.Marshal(obj) + if err != nil { + return &s3.PutObjectOutput{}, err + } return c.awsClient.PutObject( context.Background(), @@ -51,19 +51,19 @@ func (c *S3Client) Get(objectKey string) (*s3.GetObjectOutput, error) { } type resolverV2 struct { - URL string + URL string } func (r *resolverV2) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (smithyendpoints.Endpoint, error) { - if r.URL != "" { - u, err := url.Parse(r.URL) - if err != nil { - return smithyendpoints.Endpoint{}, err - } - return smithyendpoints.Endpoint{ URI: *u }, nil - } + if r.URL != "" { + u, err := url.Parse(r.URL) + if err != nil { + return smithyendpoints.Endpoint{}, err + } + return smithyendpoints.Endpoint{ URI: *u }, nil + } - return s3.NewDefaultEndpointResolverV2().ResolveEndpoint(ctx, params) + return s3.NewDefaultEndpointResolverV2().ResolveEndpoint(ctx, params) } // set endpoint to "" outside dev to use default resolver From 3fc1e0400833875d69b93d9ba35531950109fe75 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Wed, 28 Feb 2024 09:08:00 +0000 Subject: [PATCH 21/29] Get endpoint resolver working locally --- internal/objectstore/client.go | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go index c21d7879..37d8937e 100644 --- a/internal/objectstore/client.go +++ b/internal/objectstore/client.go @@ -4,13 +4,11 @@ import ( "bytes" "context" "encoding/json" - "net/url" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - smithyendpoints "github.com/aws/smithy-go/endpoints" ) type awsS3Client interface { @@ -54,6 +52,11 @@ type resolverV2 struct { URL string } +func (r *resolverV2) ResolveEndpoint(service, region string) (aws.Endpoint, error) { + return aws.Endpoint{ URL: r.URL, HostnameImmutable: true, }, nil +} + +/* func (r *resolverV2) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (smithyendpoints.Endpoint, error) { if r.URL != "" { u, err := url.Parse(r.URL) @@ -65,19 +68,30 @@ func (r *resolverV2) ResolveEndpoint(ctx context.Context, params s3.EndpointPara return s3.NewDefaultEndpointResolverV2().ResolveEndpoint(ctx, params) } +*/ // set endpoint to "" outside dev to use default resolver func NewS3Client(bucketName, endpointURL string) *S3Client { - cfg, err := config.LoadDefaultConfig(context.Background()) + var ( + cfg aws.Config + err error + awsClient *s3.Client + ) + + if endpointURL == "" { + cfg, err = config.LoadDefaultConfig(context.Background()) + } else { + cfg, err = config.LoadDefaultConfig( + context.Background(), + config.WithEndpointResolver(&resolverV2{ URL: endpointURL }), + ) + } + if err != nil { panic(err) } - awsClient := s3.NewFromConfig(cfg, func (o *s3.Options) { - o.EndpointResolverV2 = &resolverV2{ - URL: endpointURL, - } - }) + awsClient = s3.NewFromConfig(cfg) return &S3Client{ bucketName: bucketName, From e70a004bdcc1e50c40f89398b7e81bd642519d4c Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Wed, 28 Feb 2024 09:49:47 +0000 Subject: [PATCH 22/29] Go back to deprecated endpoint resolver --- internal/objectstore/client.go | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go index 37d8937e..c600f73d 100644 --- a/internal/objectstore/client.go +++ b/internal/objectstore/client.go @@ -48,34 +48,19 @@ func (c *S3Client) Get(objectKey string) (*s3.GetObjectOutput, error) { ) } -type resolverV2 struct { +type resolver struct { URL string } -func (r *resolverV2) ResolveEndpoint(service, region string) (aws.Endpoint, error) { +func (r *resolver) ResolveEndpoint(service, region string) (aws.Endpoint, error) { return aws.Endpoint{ URL: r.URL, HostnameImmutable: true, }, nil } -/* -func (r *resolverV2) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (smithyendpoints.Endpoint, error) { - if r.URL != "" { - u, err := url.Parse(r.URL) - if err != nil { - return smithyendpoints.Endpoint{}, err - } - return smithyendpoints.Endpoint{ URI: *u }, nil - } - - return s3.NewDefaultEndpointResolverV2().ResolveEndpoint(ctx, params) -} -*/ - // set endpoint to "" outside dev to use default resolver func NewS3Client(bucketName, endpointURL string) *S3Client { var ( cfg aws.Config err error - awsClient *s3.Client ) if endpointURL == "" { @@ -83,7 +68,7 @@ func NewS3Client(bucketName, endpointURL string) *S3Client { } else { cfg, err = config.LoadDefaultConfig( context.Background(), - config.WithEndpointResolver(&resolverV2{ URL: endpointURL }), + config.WithEndpointResolver(&resolver{ URL: endpointURL }), ) } @@ -91,7 +76,7 @@ func NewS3Client(bucketName, endpointURL string) *S3Client { panic(err) } - awsClient = s3.NewFromConfig(cfg) + awsClient := s3.NewFromConfig(cfg) return &S3Client{ bucketName: bucketName, From 3ea7f315bc1071bb29a45a4d3cc7be37b4585c8b Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Wed, 28 Feb 2024 10:16:55 +0000 Subject: [PATCH 23/29] Remove deprecated endpoint resolver approach --- internal/objectstore/client.go | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go index c600f73d..615f1981 100644 --- a/internal/objectstore/client.go +++ b/internal/objectstore/client.go @@ -48,30 +48,25 @@ func (c *S3Client) Get(objectKey string) (*s3.GetObjectOutput, error) { ) } -type resolver struct { - URL string -} - -func (r *resolver) ResolveEndpoint(service, region string) (aws.Endpoint, error) { - return aws.Endpoint{ URL: r.URL, HostnameImmutable: true, }, nil -} - // set endpoint to "" outside dev to use default resolver func NewS3Client(bucketName, endpointURL string) *S3Client { - var ( - cfg aws.Config - err error - ) - - if endpointURL == "" { - cfg, err = config.LoadDefaultConfig(context.Background()) - } else { - cfg, err = config.LoadDefaultConfig( - context.Background(), - config.WithEndpointResolver(&resolver{ URL: endpointURL }), + var endpointResolverWithOptions aws.EndpointResolverWithOptions + if endpointURL != "" { + endpointResolverWithOptions = aws.EndpointResolverWithOptionsFunc( + func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ URL: endpointURL, HostnameImmutable: true, }, nil + }, ) } + cfg, err := config.LoadDefaultConfig( + context.Background(), + func (o *config.LoadOptions) error { + o.EndpointResolverWithOptions = endpointResolverWithOptions + return nil + }, + ) + if err != nil { panic(err) } From cbb0563ea2c64e8135a057c760d8878948c6aee4 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Fri, 1 Mar 2024 14:03:49 +0000 Subject: [PATCH 24/29] PR review and formatting --- internal/ddb/client.go | 8 ++++---- internal/event/client.go | 10 ++++----- internal/event/client_test.go | 16 +++++++-------- internal/objectstore/client.go | 8 ++++---- internal/objectstore/client_test.go | 6 +++--- internal/shared/lpa.go | 6 ------ internal/shared/problem.go | 6 +++--- lambda/create/main.go | 12 +++++------ lambda/get/main.go | 2 +- lambda/update/main.go | 14 ++++++------- lambda/update/main_test.go | 32 ++++++++++++++--------------- 11 files changed, 56 insertions(+), 64 deletions(-) diff --git a/internal/ddb/client.go b/internal/ddb/client.go index 62091c8e..a367cc48 100644 --- a/internal/ddb/client.go +++ b/internal/ddb/client.go @@ -19,11 +19,11 @@ type Client struct { func (c *Client) PutChanges(ctx context.Context, data any, update shared.Update) error { changesItem, _ := dynamodbattribute.MarshalMap(map[string]interface{}{ - "uid": update.Uid, + "uid": update.Uid, "applied": update.Applied, - "author": update.Author, - "type": update.Type, - "change": update.Changes, + "author": update.Author, + "type": update.Type, + "change": update.Changes, }) item, err := dynamodbattribute.MarshalMap(data) diff --git a/internal/event/client.go b/internal/event/client.go index e3793318..ffd6e832 100644 --- a/internal/event/client.go +++ b/internal/event/client.go @@ -23,7 +23,7 @@ type Client struct { func NewClient(cfg aws.Config, eventBusName string) *Client { return &Client{ - svc: eventbridge.NewFromConfig(cfg, func (o *eventbridge.Options) { + svc: eventbridge.NewFromConfig(cfg, func(o *eventbridge.Options) { o.BaseEndpoint = aws.String(os.Getenv("AWS_EVENTBRIDGE_ENDPOINT")) }), eventBusName: eventBusName, @@ -40,13 +40,13 @@ func (c *Client) send(ctx context.Context, eventType string, detail any) error { if err != nil { return err } - + _, err = c.svc.PutEvents(ctx, &eventbridge.PutEventsInput{ Entries: []types.PutEventsRequestEntry{{ EventBusName: aws.String(c.eventBusName), - Source: aws.String(source), - DetailType: aws.String(eventType), - Detail: aws.String(string(v)), + Source: aws.String(source), + DetailType: aws.String(eventType), + Detail: aws.String(string(v)), }}, }) diff --git a/internal/event/client_test.go b/internal/event/client_test.go index ffee2f20..66242cc2 100644 --- a/internal/event/client_test.go +++ b/internal/event/client_test.go @@ -28,18 +28,18 @@ func TestClientSendEvent(t *testing.T) { ctx := context.Background() expectedError := errors.New("err") - event := LpaUpdated{ Uid: "M-1234-1234-1234", ChangeType: "CREATED" } + event := LpaUpdated{Uid: "M-1234-1234-1234", ChangeType: "CREATED"} data, _ := json.Marshal(event) mockClient := &mockEventBridgeClient{} mockClient.On("PutEvents", mock.Anything, &eventbridge.PutEventsInput{ - Entries: []types.PutEventsRequestEntry{{ - EventBusName: aws.String("my-bus"), - Source: aws.String("opg.poas.lpastore"), - DetailType: aws.String("lpa-updated"), - Detail: aws.String(string(data)), - }}, - }). + Entries: []types.PutEventsRequestEntry{{ + EventBusName: aws.String("my-bus"), + Source: aws.String("opg.poas.lpastore"), + DetailType: aws.String("lpa-updated"), + Detail: aws.String(string(data)), + }}, + }). Return(nil, expectedError) svc := &Client{svc: mockClient, eventBusName: "my-bus"} diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go index 615f1981..58a1a91f 100644 --- a/internal/objectstore/client.go +++ b/internal/objectstore/client.go @@ -24,7 +24,7 @@ type S3Client struct { func (c *S3Client) Put(objectKey string, obj any) (*s3.PutObjectOutput, error) { b, err := json.Marshal(obj) if err != nil { - return &s3.PutObjectOutput{}, err + return nil, err } return c.awsClient.PutObject( @@ -54,14 +54,14 @@ func NewS3Client(bucketName, endpointURL string) *S3Client { if endpointURL != "" { endpointResolverWithOptions = aws.EndpointResolverWithOptionsFunc( func(service, region string, options ...interface{}) (aws.Endpoint, error) { - return aws.Endpoint{ URL: endpointURL, HostnameImmutable: true, }, nil + return aws.Endpoint{URL: endpointURL, HostnameImmutable: true}, nil }, ) } cfg, err := config.LoadDefaultConfig( context.Background(), - func (o *config.LoadOptions) error { + func(o *config.LoadOptions) error { o.EndpointResolverWithOptions = endpointResolverWithOptions return nil }, @@ -75,6 +75,6 @@ func NewS3Client(bucketName, endpointURL string) *S3Client { return &S3Client{ bucketName: bucketName, - awsClient: awsClient, + awsClient: awsClient, } } diff --git a/internal/objectstore/client_test.go b/internal/objectstore/client_test.go index 60cdd32a..bc615a1a 100644 --- a/internal/objectstore/client_test.go +++ b/internal/objectstore/client_test.go @@ -29,10 +29,10 @@ func TestPut(t *testing.T) { c := S3Client{ bucketName: "bucket1", - awsClient: &client, + awsClient: &client, } - _, err := c.Put("anobject", struct{ID int}{ID: 1}) + _, err := c.Put("anobject", struct{ ID int }{ID: 1}) assert.Equal(t, nil, err) client.AssertExpectations(t) @@ -44,7 +44,7 @@ func TestGet(t *testing.T) { c := S3Client{ bucketName: "bucket1", - awsClient: &client, + awsClient: &client, } _, err := c.Get("anotherobject") diff --git a/internal/shared/lpa.go b/internal/shared/lpa.go index 81273247..0b8a5c3a 100644 --- a/internal/shared/lpa.go +++ b/internal/shared/lpa.go @@ -1,8 +1,6 @@ package shared import ( - "encoding/json" - "io" "time" ) @@ -51,7 +49,3 @@ const ( LpaStatusProcessing = LpaStatus("processing") LpaStatusRegistered = LpaStatus("registered") ) - -func (l *Lpa) FromJSON(reader io.Reader) error { - return json.NewDecoder(reader).Decode(l) -} diff --git a/internal/shared/problem.go b/internal/shared/problem.go index 890a1883..bbe4e194 100644 --- a/internal/shared/problem.go +++ b/internal/shared/problem.go @@ -16,9 +16,9 @@ type FieldError struct { } type Problem struct { - StatusCode int `json:"-"` - Code string `json:"code"` - Detail string `json:"detail"` + StatusCode int `json:"-"` + Code string `json:"code"` + Detail string `json:"detail"` Errors []FieldError `json:"errors,omitempty"` } diff --git a/lambda/create/main.go b/lambda/create/main.go index 7367e444..dd0276ea 100644 --- a/lambda/create/main.go +++ b/lambda/create/main.go @@ -85,8 +85,7 @@ func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequ } // validation - errs := Validate(input) - if len(errs) > 0 { + if errs := Validate(input); len(errs) > 0 { problem := shared.ProblemInvalidRequest problem.Errors = errs @@ -99,8 +98,7 @@ func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequ data.UpdatedAt = time.Now() // save - err = l.store.Put(ctx, data) - if err != nil { + if err = l.store.Put(ctx, data); err != nil { l.logger.Print(err) return shared.ProblemInternalServerError.Respond() } @@ -116,7 +114,7 @@ func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequ // send lpa-updated event err = l.eventClient.SendLpaUpdated(ctx, event.LpaUpdated{ - Uid: uid, + Uid: uid, ChangeType: "CREATED", }) @@ -136,7 +134,7 @@ func main() { ctx := context.Background() awsConfig, err := config.LoadDefaultConfig(ctx) if err != nil { - logger.Print("Failed to load configuration:", err) + logger.Print("Failed to load configuration:", err) } l := &Lambda{ @@ -151,7 +149,7 @@ func main() { os.Getenv("AWS_S3_ENDPOINT"), ), verifier: shared.NewJWTVerifier(), - logger: logger, + logger: logger, } lambda.Start(l.HandleEvent) diff --git a/lambda/get/main.go b/lambda/get/main.go index f0cf203d..a5fe6781 100644 --- a/lambda/get/main.go +++ b/lambda/get/main.go @@ -73,7 +73,7 @@ func (l *Lambda) HandleEvent(ctx context.Context, event events.APIGatewayProxyRe func main() { l := &Lambda{ - store: ddb.New( + store: ddb.New( os.Getenv("AWS_DYNAMODB_ENDPOINT"), os.Getenv("DDB_TABLE_NAME_DEEDS"), os.Getenv("DDB_TABLE_NAME_CHANGES"), diff --git a/lambda/update/main.go b/lambda/update/main.go index b7d161d4..b5dac94e 100644 --- a/lambda/update/main.go +++ b/lambda/update/main.go @@ -34,10 +34,10 @@ type Verifier interface { } type Lambda struct { - eventClient EventClient - store Store - verifier Verifier - logger Logger + eventClient EventClient + store Store + verifier Verifier + logger Logger } func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) { @@ -103,7 +103,7 @@ func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequ // send lpa-updated event err = l.eventClient.SendLpaUpdated(ctx, event.LpaUpdated{ - Uid: lpa.Uid, + Uid: lpa.Uid, ChangeType: "UPDATED", }) @@ -122,12 +122,12 @@ func main() { ctx := context.Background() awsConfig, err := config.LoadDefaultConfig(ctx) if err != nil { - logger.Print("Failed to load configuration:", err) + logger.Print("Failed to load configuration:", err) } l := &Lambda{ eventClient: event.NewClient(awsConfig, os.Getenv("EVENT_BUS_NAME")), - store: ddb.New( + store: ddb.New( os.Getenv("AWS_DYNAMODB_ENDPOINT"), os.Getenv("DDB_TABLE_NAME_DEEDS"), os.Getenv("DDB_TABLE_NAME_CHANGES"), diff --git a/lambda/update/main_test.go b/lambda/update/main_test.go index 75bdfc51..0b232b50 100644 --- a/lambda/update/main_test.go +++ b/lambda/update/main_test.go @@ -10,10 +10,10 @@ import ( "time" "github.com/aws/aws-lambda-go/events" + "github.com/golang-jwt/jwt/v5" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/uuid" - "github.com/golang-jwt/jwt/v5" "github.com/ministryofjustice/opg-data-lpa-store/internal/event" "github.com/ministryofjustice/opg-data-lpa-store/internal/shared" "github.com/ministryofjustice/opg-go-common/logging" @@ -61,7 +61,7 @@ func (m *mockStore) PutChanges(ctx context.Context, data any, update shared.Upda type mockVerifier struct { claims shared.LpaStoreClaims - err error + err error } func (m *mockVerifier) VerifyHeader(events.APIGatewayProxyRequest) (*shared.LpaStoreClaims, error) { @@ -74,15 +74,15 @@ func TestHandleEvent(t *testing.T) { client.On("SendLpaUpdated", mock.Anything, mock.Anything).Return(nil) l := Lambda{ eventClient: &client, - store: store, - verifier: &mockVerifier{ + store: store, + verifier: &mockVerifier{ claims: shared.LpaStoreClaims{ RegisteredClaims: jwt.RegisteredClaims{ Subject: "1234", }, }, }, - logger: logging.New(io.Discard, ""), + logger: logging.New(io.Discard, ""), } resp, err := l.HandleEvent(context.Background(), events.APIGatewayProxyRequest{ @@ -108,9 +108,9 @@ func TestHandleEvent(t *testing.T) { assert.True(t, cmp.Equal( shared.Update{ - Uid: "1", + Uid: "1", Author: "1234", - Type: "CERTIFICATE_PROVIDER_SIGN", + Type: "CERTIFICATE_PROVIDER_SIGN", Changes: []shared.Change{ shared.Change{ Key: "/certificateProvider/signedAt", @@ -132,9 +132,9 @@ func TestHandleEvent(t *testing.T) { func TestHandleEventWhenUnknownType(t *testing.T) { l := Lambda{ - store: &mockStore{get: shared.Lpa{Uid: "1"}}, - verifier: &mockVerifier{}, - logger: logging.New(io.Discard, ""), + store: &mockStore{get: shared.Lpa{Uid: "1"}}, + verifier: &mockVerifier{}, + logger: logging.New(io.Discard, ""), } resp, err := l.HandleEvent(context.Background(), events.APIGatewayProxyRequest{ @@ -147,9 +147,9 @@ func TestHandleEventWhenUnknownType(t *testing.T) { func TestHandleEventWhenUpdateInvalid(t *testing.T) { l := Lambda{ - store: &mockStore{get: shared.Lpa{Uid: "1"}}, - verifier: &mockVerifier{}, - logger: logging.New(io.Discard, ""), + store: &mockStore{get: shared.Lpa{Uid: "1"}}, + verifier: &mockVerifier{}, + logger: logging.New(io.Discard, ""), } resp, err := l.HandleEvent(context.Background(), events.APIGatewayProxyRequest{ @@ -225,15 +225,15 @@ func TestHandleEventWhenSendLpaUpdatedFailed(t *testing.T) { l := Lambda{ eventClient: &client, - store: store, - verifier: &mockVerifier{ + store: store, + verifier: &mockVerifier{ claims: shared.LpaStoreClaims{ RegisteredClaims: jwt.RegisteredClaims{ Subject: "1234", }, }, }, - logger: &logger, + logger: &logger, } resp, err := l.HandleEvent(context.Background(), events.APIGatewayProxyRequest{ From bcca6e2f97fb1d3566a5e6209b5ef34f5ffced41 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Fri, 1 Mar 2024 14:09:30 +0000 Subject: [PATCH 25/29] Remove unused funcs and return values --- internal/objectstore/client.go | 17 ++++------------- lambda/create/main.go | 5 ++--- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go index 58a1a91f..0720027e 100644 --- a/internal/objectstore/client.go +++ b/internal/objectstore/client.go @@ -13,7 +13,6 @@ import ( type awsS3Client interface { PutObject(ctx context.Context, input *s3.PutObjectInput, opts ...func(*s3.Options)) (*s3.PutObjectOutput, error) - GetObject(ctx context.Context, input *s3.GetObjectInput, opts ...func(*s3.Options)) (*s3.GetObjectOutput, error) } type S3Client struct { @@ -21,13 +20,13 @@ type S3Client struct { awsClient awsS3Client } -func (c *S3Client) Put(objectKey string, obj any) (*s3.PutObjectOutput, error) { +func (c *S3Client) Put(objectKey string, obj any) error { b, err := json.Marshal(obj) if err != nil { - return nil, err + return err } - return c.awsClient.PutObject( + _, err = c.awsClient.PutObject( context.Background(), &s3.PutObjectInput{ Bucket: aws.String(c.bucketName), @@ -36,16 +35,8 @@ func (c *S3Client) Put(objectKey string, obj any) (*s3.PutObjectOutput, error) { ServerSideEncryption: types.ServerSideEncryptionAwsKms, }, ) -} -func (c *S3Client) Get(objectKey string) (*s3.GetObjectOutput, error) { - return c.awsClient.GetObject( - context.Background(), - &s3.GetObjectInput{ - Bucket: aws.String(c.bucketName), - Key: aws.String(objectKey), - }, - ) + return err } // set endpoint to "" outside dev to use default resolver diff --git a/lambda/create/main.go b/lambda/create/main.go index dd0276ea..04fb1064 100644 --- a/lambda/create/main.go +++ b/lambda/create/main.go @@ -32,7 +32,7 @@ type Store interface { } type S3Client interface { - Put(objectKey string, obj any) (*s3.PutObjectOutput, error) + Put(objectKey string, obj any) error } type Verifier interface { @@ -105,9 +105,8 @@ func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequ // save to static storage as JSON objectKey := fmt.Sprintf("%s/donor-executed-lpa.json", data.Uid) - _, err = l.staticLpaStorage.Put(objectKey, data) - if err != nil { + if err = l.staticLpaStorage.Put(objectKey, data); err != nil { l.logger.Print(err) return shared.ProblemInternalServerError.Respond() } From f5f55ba9600bb4333bd2e69420a06106b9ef17ab Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Fri, 1 Mar 2024 15:01:33 +0000 Subject: [PATCH 26/29] Move config out of client into lambda --- internal/objectstore/client.go | 26 ++------------------------ lambda/create/main.go | 25 +++++++++++++++++++++---- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go index 0720027e..a30e9ebc 100644 --- a/internal/objectstore/client.go +++ b/internal/objectstore/client.go @@ -6,7 +6,6 @@ import ( "encoding/json" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" ) @@ -40,29 +39,8 @@ func (c *S3Client) Put(objectKey string, obj any) error { } // set endpoint to "" outside dev to use default resolver -func NewS3Client(bucketName, endpointURL string) *S3Client { - var endpointResolverWithOptions aws.EndpointResolverWithOptions - if endpointURL != "" { - endpointResolverWithOptions = aws.EndpointResolverWithOptionsFunc( - func(service, region string, options ...interface{}) (aws.Endpoint, error) { - return aws.Endpoint{URL: endpointURL, HostnameImmutable: true}, nil - }, - ) - } - - cfg, err := config.LoadDefaultConfig( - context.Background(), - func(o *config.LoadOptions) error { - o.EndpointResolverWithOptions = endpointResolverWithOptions - return nil - }, - ) - - if err != nil { - panic(err) - } - - awsClient := s3.NewFromConfig(cfg) +func NewS3Client(awsConfig aws.Config, bucketName string) *S3Client { + awsClient := s3.NewFromConfig(awsConfig) return &S3Client{ bucketName: bucketName, diff --git a/lambda/create/main.go b/lambda/create/main.go index 04fb1064..c357a2d6 100644 --- a/lambda/create/main.go +++ b/lambda/create/main.go @@ -9,8 +9,8 @@ import ( "github.com/aws/aws-lambda-go/events" "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/ministryofjustice/opg-data-lpa-store/internal/ddb" "github.com/ministryofjustice/opg-data-lpa-store/internal/event" "github.com/ministryofjustice/opg-data-lpa-store/internal/objectstore" @@ -130,8 +130,25 @@ func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequ func main() { logger := logging.New(os.Stdout, "opg-data-lpa-store") - ctx := context.Background() - awsConfig, err := config.LoadDefaultConfig(ctx) + endpointURL := os.Getenv("AWS_S3_ENDPOINT") + + var endpointResolverWithOptions aws.EndpointResolverWithOptions + if endpointURL != "" { + endpointResolverWithOptions = aws.EndpointResolverWithOptionsFunc( + func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{URL: endpointURL, HostnameImmutable: true}, nil + }, + ) + } + + awsConfig, err := config.LoadDefaultConfig( + context.Background(), + func(o *config.LoadOptions) error { + o.EndpointResolverWithOptions = endpointResolverWithOptions + return nil + }, + ) + if err != nil { logger.Print("Failed to load configuration:", err) } @@ -144,8 +161,8 @@ func main() { os.Getenv("DDB_TABLE_NAME_CHANGES"), ), staticLpaStorage: objectstore.NewS3Client( + awsConfig, os.Getenv("S3_BUCKET_NAME_ORIGINAL"), - os.Getenv("AWS_S3_ENDPOINT"), ), verifier: shared.NewJWTVerifier(), logger: logger, From 0766138cfc429bcd1e9acec4dad1c7bbeb513ca7 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Fri, 1 Mar 2024 15:26:17 +0000 Subject: [PATCH 27/29] Pass context from lambda into S3 client --- internal/objectstore/client.go | 5 ++--- lambda/create/main.go | 6 ++++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/objectstore/client.go b/internal/objectstore/client.go index a30e9ebc..799d8bf5 100644 --- a/internal/objectstore/client.go +++ b/internal/objectstore/client.go @@ -19,14 +19,14 @@ type S3Client struct { awsClient awsS3Client } -func (c *S3Client) Put(objectKey string, obj any) error { +func (c *S3Client) Put(ctx context.Context, objectKey string, obj any) error { b, err := json.Marshal(obj) if err != nil { return err } _, err = c.awsClient.PutObject( - context.Background(), + ctx, &s3.PutObjectInput{ Bucket: aws.String(c.bucketName), Key: aws.String(objectKey), @@ -38,7 +38,6 @@ func (c *S3Client) Put(objectKey string, obj any) error { return err } -// set endpoint to "" outside dev to use default resolver func NewS3Client(awsConfig aws.Config, bucketName string) *S3Client { awsClient := s3.NewFromConfig(awsConfig) diff --git a/lambda/create/main.go b/lambda/create/main.go index c357a2d6..226721b7 100644 --- a/lambda/create/main.go +++ b/lambda/create/main.go @@ -32,7 +32,7 @@ type Store interface { } type S3Client interface { - Put(objectKey string, obj any) error + Put(ctx context.Context, objectKey string, obj any) error } type Verifier interface { @@ -106,7 +106,7 @@ func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequ // save to static storage as JSON objectKey := fmt.Sprintf("%s/donor-executed-lpa.json", data.Uid) - if err = l.staticLpaStorage.Put(objectKey, data); err != nil { + if err = l.staticLpaStorage.Put(ctx, objectKey, data); err != nil { l.logger.Print(err) return shared.ProblemInternalServerError.Respond() } @@ -130,6 +130,8 @@ func (l *Lambda) HandleEvent(ctx context.Context, req events.APIGatewayProxyRequ func main() { logger := logging.New(os.Stdout, "opg-data-lpa-store") + + // set endpoint to "" outside dev to use default AWS resolver endpointURL := os.Getenv("AWS_S3_ENDPOINT") var endpointResolverWithOptions aws.EndpointResolverWithOptions From 49b4145845f0deb57c9b5c7817e1560d7a7191d4 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Fri, 1 Mar 2024 15:57:36 +0000 Subject: [PATCH 28/29] Fix broken tests --- internal/objectstore/client_test.go | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/internal/objectstore/client_test.go b/internal/objectstore/client_test.go index bc615a1a..550cbed5 100644 --- a/internal/objectstore/client_test.go +++ b/internal/objectstore/client_test.go @@ -18,36 +18,16 @@ func (m *mockAwsClient) PutObject(ctx context.Context, input *s3.PutObjectInput, return args.Get(0).(*s3.PutObjectOutput), args.Error(1) } -func (m *mockAwsClient) GetObject(ctx context.Context, input *s3.GetObjectInput, opts ...func(*s3.Options)) (*s3.GetObjectOutput, error) { - args := m.Called(ctx, input) - return args.Get(0).(*s3.GetObjectOutput), args.Error(1) -} - func TestPut(t *testing.T) { client := mockAwsClient{} - client.On("PutObject", mock.Anything, mock.Anything).Return(&s3.PutObjectOutput{}, nil) - - c := S3Client{ - bucketName: "bucket1", - awsClient: &client, - } - - _, err := c.Put("anobject", struct{ ID int }{ID: 1}) - - assert.Equal(t, nil, err) - client.AssertExpectations(t) -} - -func TestGet(t *testing.T) { - client := mockAwsClient{} - client.On("GetObject", mock.Anything, mock.Anything).Return(&s3.GetObjectOutput{}, nil) + client.On("PutObject", mock.Anything, mock.Anything, mock.Anything).Return(&s3.PutObjectOutput{}, nil) c := S3Client{ bucketName: "bucket1", awsClient: &client, } - _, err := c.Get("anotherobject") + err := c.Put(context.Background(), "anobject", struct{ ID int }{ID: 1}) assert.Equal(t, nil, err) client.AssertExpectations(t) From c6d78304ce9f4d7163cad31b3276b3eff1ed94c6 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Fri, 1 Mar 2024 16:17:59 +0000 Subject: [PATCH 29/29] Keep S3 config separate from other service config --- lambda/create/main.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lambda/create/main.go b/lambda/create/main.go index 226721b7..1b2f82bc 100644 --- a/lambda/create/main.go +++ b/lambda/create/main.go @@ -143,27 +143,33 @@ func main() { ) } - awsConfig, err := config.LoadDefaultConfig( - context.Background(), + ctx := context.Background() + + eventClientConfig, err := config.LoadDefaultConfig(ctx) + if err != nil { + logger.Print("Failed to load event client configuration:", err) + } + + s3Config, err := config.LoadDefaultConfig( + ctx, func(o *config.LoadOptions) error { o.EndpointResolverWithOptions = endpointResolverWithOptions return nil }, ) - if err != nil { - logger.Print("Failed to load configuration:", err) + logger.Print("Failed to load S3 configuration:", err) } l := &Lambda{ - eventClient: event.NewClient(awsConfig, os.Getenv("EVENT_BUS_NAME")), + eventClient: event.NewClient(eventClientConfig, os.Getenv("EVENT_BUS_NAME")), store: ddb.New( os.Getenv("AWS_DYNAMODB_ENDPOINT"), os.Getenv("DDB_TABLE_NAME_DEEDS"), os.Getenv("DDB_TABLE_NAME_CHANGES"), ), staticLpaStorage: objectstore.NewS3Client( - awsConfig, + s3Config, os.Getenv("S3_BUCKET_NAME_ORIGINAL"), ), verifier: shared.NewJWTVerifier(),