From 38791cf25ac16679e01a8e14c78cbe5f8f3aaf74 Mon Sep 17 00:00:00 2001 From: nnnkkk7 Date: Sun, 19 Nov 2023 15:26:55 +0900 Subject: [PATCH 1/4] add ephemeralStorage option to lambda config Signed-off-by: nnnkkk7 Signed-off-by: nnnkkk7 --- .../defining-app-configuration/lambda.md | 4 ++ .../piped/platformprovider/lambda/client.go | 6 ++ .../piped/platformprovider/lambda/function.go | 55 +++++++++++++------ .../platformprovider/lambda/function_test.go | 49 +++++++++++++++++ 4 files changed, 96 insertions(+), 18 deletions(-) diff --git a/docs/content/en/docs-dev/user-guide/managing-application/defining-app-configuration/lambda.md b/docs/content/en/docs-dev/user-guide/managing-application/defining-app-configuration/lambda.md index c58fe6fbff..d6bf0a15e8 100644 --- a/docs/content/en/docs-dev/user-guide/managing-application/defining-app-configuration/lambda.md +++ b/docs/content/en/docs-dev/user-guide/managing-application/defining-app-configuration/lambda.md @@ -34,6 +34,10 @@ spec: app: simple environments: FOO: bar + # ephemeralStorage is optional value. If you define a ephemeral storage to lambda, you can + # use this field. The value must be in between 512 to 10240 MB. + ephemeralStorage: + size: 512 # vpcConfig is optional value. If you define a vpc configuration to lambda, you can # use this field. vpcConfig: diff --git a/pkg/app/piped/platformprovider/lambda/client.go b/pkg/app/piped/platformprovider/lambda/client.go index d015c167ae..c243efc0dc 100644 --- a/pkg/app/piped/platformprovider/lambda/client.go +++ b/pkg/app/piped/platformprovider/lambda/client.go @@ -126,6 +126,9 @@ func (c *client) CreateFunction(ctx context.Context, fm FunctionManifest) error } input.Architectures = architectures } + if fm.Spec.EphemeralStorage.Size != 0 { + input.EphemeralStorage.Size = aws.Int32(fm.Spec.EphemeralStorage.Size) + } if fm.Spec.VPCConfig != nil { input.VpcConfig = &types.VpcConfig{ SecurityGroupIds: fm.Spec.VPCConfig.SecurityGroupIDs, @@ -278,6 +281,9 @@ func (c *client) updateFunctionConfiguration(ctx context.Context, fm FunctionMan if fm.Spec.Handler != "" { configInput.Handler = aws.String(fm.Spec.Handler) } + if fm.Spec.EphemeralStorage.Size != 0 { + configInput.EphemeralStorage.Size = aws.Int32(fm.Spec.EphemeralStorage.Size) + } if fm.Spec.VPCConfig != nil { configInput.VpcConfig = &types.VpcConfig{ SecurityGroupIds: fm.Spec.VPCConfig.SecurityGroupIDs, diff --git a/pkg/app/piped/platformprovider/lambda/function.go b/pkg/app/piped/platformprovider/lambda/function.go index a6e4d70206..322bc2a0a2 100644 --- a/pkg/app/piped/platformprovider/lambda/function.go +++ b/pkg/app/piped/platformprovider/lambda/function.go @@ -30,9 +30,11 @@ const ( functionManifestKind = "LambdaFunction" // Memory and Timeout lower and upper limit as noted via // https://docs.aws.amazon.com/sdk-for-go/api/service/lambda/#UpdateFunctionConfigurationInput - memoryLowerLimit = 1 - timeoutLowerLimit = 1 - timeoutUpperLimit = 900 + memoryLowerLimit = 1 + timeoutLowerLimit = 1 + timeoutUpperLimit = 900 + ephemeralStorageLowerLimit = 512 + ephemeralStorageUpperLimit = 10240 ) type FunctionManifest struct { @@ -56,21 +58,22 @@ func (fm *FunctionManifest) validate() error { // FunctionManifestSpec contains configuration for LambdaFunction. type FunctionManifestSpec struct { - Name string `json:"name"` - Role string `json:"role"` - ImageURI string `json:"image"` - S3Bucket string `json:"s3Bucket"` - S3Key string `json:"s3Key"` - S3ObjectVersion string `json:"s3ObjectVersion"` - SourceCode SourceCode `json:"source"` - Handler string `json:"handler"` - Architectures []Architecture `json:"architectures,omitempty"` - Runtime string `json:"runtime"` - Memory int32 `json:"memory"` - Timeout int32 `json:"timeout"` - Tags map[string]string `json:"tags,omitempty"` - Environments map[string]string `json:"environments,omitempty"` - VPCConfig *VPCConfig `json:"vpcConfig,omitempty"` + Name string `json:"name"` + Role string `json:"role"` + ImageURI string `json:"image"` + S3Bucket string `json:"s3Bucket"` + S3Key string `json:"s3Key"` + S3ObjectVersion string `json:"s3ObjectVersion"` + SourceCode SourceCode `json:"source"` + Handler string `json:"handler"` + Architectures []Architecture `json:"architectures,omitempty"` + EphemeralStorage *EphemeralStorage `json:"ephemeralStorage,omitempty"` + Runtime string `json:"runtime"` + Memory int32 `json:"memory"` + Timeout int32 `json:"timeout"` + Tags map[string]string `json:"tags,omitempty"` + Environments map[string]string `json:"environments,omitempty"` + VPCConfig *VPCConfig `json:"vpcConfig,omitempty"` } type VPCConfig struct { @@ -100,6 +103,11 @@ func (fmp FunctionManifestSpec) validate() error { return fmt.Errorf("architecture is invalid: %w", err) } } + if fmp.EphemeralStorage != nil && fmp.EphemeralStorage.Size != 0 { + if err := fmp.EphemeralStorage.validate(); err != nil { + return fmt.Errorf("ephemeral storage is invalid: %w", err) + } + } if fmp.Role == "" { return fmt.Errorf("role is missing") } @@ -139,6 +147,17 @@ func (a Architecture) validate() error { return nil } +type EphemeralStorage struct { + Size int32 `json:"size,omitempty"` +} + +func (es EphemeralStorage) validate() error { + if es.Size < ephemeralStorageLowerLimit || es.Size > ephemeralStorageUpperLimit { + return fmt.Errorf("ephemeral storage is out of range") + } + return nil +} + func loadFunctionManifest(path string) (FunctionManifest, error) { data, err := os.ReadFile(path) if err != nil { diff --git a/pkg/app/piped/platformprovider/lambda/function_test.go b/pkg/app/piped/platformprovider/lambda/function_test.go index eefbb3ed2b..8c9a5134a4 100644 --- a/pkg/app/piped/platformprovider/lambda/function_test.go +++ b/pkg/app/piped/platformprovider/lambda/function_test.go @@ -99,6 +99,36 @@ func TestParseFunctionManifest(t *testing.T) { }, wantErr: false, }, + { + name: "correct config for LambdaFunction with specifying ephemeral storage", + data: `{ + "apiVersion": "pipecd.dev/v1beta1", + "kind": "LambdaFunction", + "spec": { + "name": "SimpleFunction", + "role": "arn:aws:iam::xxxxx:role/lambda-role", + "memory": 128, + "timeout": 5, + "image": "ecr.region.amazonaws.com/lambda-simple-function:v0.0.1", + "ephemeralStorage": { + size: 512 + } + } +}`, + wantSpec: FunctionManifest{ + Kind: "LambdaFunction", + APIVersion: "pipecd.dev/v1beta1", + Spec: FunctionManifestSpec{ + Name: "SimpleFunction", + Role: "arn:aws:iam::xxxxx:role/lambda-role", + Memory: 128, + Timeout: 5, + ImageURI: "ecr.region.amazonaws.com/lambda-simple-function:v0.0.1", + EphemeralStorage: &EphemeralStorage{Size: 512}, + }, + }, + wantErr: false, + }, { name: "correct config for LambdaFunction with specifying vpc config", data: `{ @@ -170,6 +200,25 @@ func TestParseFunctionManifest(t *testing.T) { "timeout": 1000, "image": "ecr.region.amazonaws.com/lambda-simple-function:v0.0.1" } +}`, + wantSpec: FunctionManifest{}, + wantErr: true, + }, + { + name: "invalid ephemeral storage value", + data: `{ + "apiVersion": "pipecd.dev/v1beta1", + "kind": "LambdaFunction", + "spec": { + "name": "SimpleFunction", + "role": "arn:aws:iam::xxxxx:role/lambda-role", + "memory": 128, + "timeout": 5, + "image": "ecr.region.amazonaws.com/lambda-simple-function:v0.0.1", + "ephemeralStorage": { + size: 511 + } + } }`, wantSpec: FunctionManifest{}, wantErr: true, From caf132172af73103066822359604429ef3840f74 Mon Sep 17 00:00:00 2001 From: nnnkkk7 Date: Wed, 22 Nov 2023 08:47:02 +0900 Subject: [PATCH 2/4] add nil pointer check Signed-off-by: nnnkkk7 --- pkg/app/piped/platformprovider/lambda/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/app/piped/platformprovider/lambda/client.go b/pkg/app/piped/platformprovider/lambda/client.go index c243efc0dc..53dd71e119 100644 --- a/pkg/app/piped/platformprovider/lambda/client.go +++ b/pkg/app/piped/platformprovider/lambda/client.go @@ -126,7 +126,7 @@ func (c *client) CreateFunction(ctx context.Context, fm FunctionManifest) error } input.Architectures = architectures } - if fm.Spec.EphemeralStorage.Size != 0 { + if fm.Spec.EphemeralStorage != nil && fm.Spec.EphemeralStorage.Size != 0 { input.EphemeralStorage.Size = aws.Int32(fm.Spec.EphemeralStorage.Size) } if fm.Spec.VPCConfig != nil { @@ -281,7 +281,7 @@ func (c *client) updateFunctionConfiguration(ctx context.Context, fm FunctionMan if fm.Spec.Handler != "" { configInput.Handler = aws.String(fm.Spec.Handler) } - if fm.Spec.EphemeralStorage.Size != 0 { + if fm.Spec.EphemeralStorage != nil && fm.Spec.EphemeralStorage.Size != 0 { configInput.EphemeralStorage.Size = aws.Int32(fm.Spec.EphemeralStorage.Size) } if fm.Spec.VPCConfig != nil { From 639d0f448b035b82ea22525f618da90fc40a52ec Mon Sep 17 00:00:00 2001 From: nnnkkk7 Date: Wed, 22 Nov 2023 08:49:04 +0900 Subject: [PATCH 3/4] add EphemeralStorage input to CreateFunctionFromSource Signed-off-by: nnnkkk7 --- pkg/app/piped/platformprovider/lambda/client.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/app/piped/platformprovider/lambda/client.go b/pkg/app/piped/platformprovider/lambda/client.go index 53dd71e119..c4d6fd14ce 100644 --- a/pkg/app/piped/platformprovider/lambda/client.go +++ b/pkg/app/piped/platformprovider/lambda/client.go @@ -182,7 +182,9 @@ func (c *client) CreateFunctionFromSource(ctx context.Context, fm FunctionManife Variables: fm.Spec.Environments, }, } - + if fm.Spec.EphemeralStorage != nil && fm.Spec.EphemeralStorage.Size != 0 { + input.EphemeralStorage.Size = aws.Int32(fm.Spec.EphemeralStorage.Size) + } _, err = c.client.CreateFunction(ctx, input) if err != nil { return fmt.Errorf("failed to create Lambda function %s: %w", fm.Spec.Name, err) From 8af45bb9ffde7b091a96363a0c8c30d2f2f99c61 Mon Sep 17 00:00:00 2001 From: nnnkkk7 Date: Fri, 24 Nov 2023 09:51:14 +0900 Subject: [PATCH 4/4] fix condition Signed-off-by: nnnkkk7 --- pkg/app/piped/platformprovider/lambda/client.go | 6 +++--- pkg/app/piped/platformprovider/lambda/function.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/app/piped/platformprovider/lambda/client.go b/pkg/app/piped/platformprovider/lambda/client.go index c4d6fd14ce..607b110a0b 100644 --- a/pkg/app/piped/platformprovider/lambda/client.go +++ b/pkg/app/piped/platformprovider/lambda/client.go @@ -126,7 +126,7 @@ func (c *client) CreateFunction(ctx context.Context, fm FunctionManifest) error } input.Architectures = architectures } - if fm.Spec.EphemeralStorage != nil && fm.Spec.EphemeralStorage.Size != 0 { + if fm.Spec.EphemeralStorage != nil { input.EphemeralStorage.Size = aws.Int32(fm.Spec.EphemeralStorage.Size) } if fm.Spec.VPCConfig != nil { @@ -182,7 +182,7 @@ func (c *client) CreateFunctionFromSource(ctx context.Context, fm FunctionManife Variables: fm.Spec.Environments, }, } - if fm.Spec.EphemeralStorage != nil && fm.Spec.EphemeralStorage.Size != 0 { + if fm.Spec.EphemeralStorage != nil { input.EphemeralStorage.Size = aws.Int32(fm.Spec.EphemeralStorage.Size) } _, err = c.client.CreateFunction(ctx, input) @@ -283,7 +283,7 @@ func (c *client) updateFunctionConfiguration(ctx context.Context, fm FunctionMan if fm.Spec.Handler != "" { configInput.Handler = aws.String(fm.Spec.Handler) } - if fm.Spec.EphemeralStorage != nil && fm.Spec.EphemeralStorage.Size != 0 { + if fm.Spec.EphemeralStorage != nil { configInput.EphemeralStorage.Size = aws.Int32(fm.Spec.EphemeralStorage.Size) } if fm.Spec.VPCConfig != nil { diff --git a/pkg/app/piped/platformprovider/lambda/function.go b/pkg/app/piped/platformprovider/lambda/function.go index 322bc2a0a2..51e8448311 100644 --- a/pkg/app/piped/platformprovider/lambda/function.go +++ b/pkg/app/piped/platformprovider/lambda/function.go @@ -103,7 +103,7 @@ func (fmp FunctionManifestSpec) validate() error { return fmt.Errorf("architecture is invalid: %w", err) } } - if fmp.EphemeralStorage != nil && fmp.EphemeralStorage.Size != 0 { + if fmp.EphemeralStorage != nil { if err := fmp.EphemeralStorage.validate(); err != nil { return fmt.Errorf("ephemeral storage is invalid: %w", err) }