From 76593a9b35dfb8462258e6eab862a497d1325273 Mon Sep 17 00:00:00 2001 From: Marcos Diez Date: Thu, 26 Sep 2024 05:29:53 -0300 Subject: [PATCH] TargetGroupBindings can now manipulate target groups from different aws accounts #3691 --- .../elbv2/v1beta1/targetgroupbinding_types.go | 8 ++ controllers/ingress/group_controller.go | 4 +- controllers/service/service_controller.go | 4 +- docs/deploy/installation.md | 63 +++++++- docs/guide/targetgroupbinding/spec.md | 21 ++- .../targetgroupbinding/targetgroupbinding.md | 22 +++ go.mod | 8 +- go.sum | 10 ++ pkg/aws/cloud.go | 136 +++++++++++++----- pkg/aws/services/cloudInterface.go | 34 +++++ pkg/aws/services/elbv2.go | 14 +- pkg/aws/services/elbv2_mocks.go | 14 ++ pkg/deploy/stack_deployer.go | 7 +- pkg/targetgroupbinding/resource_manager.go | 65 ++++++--- pkg/targetgroupbinding/targets_manager.go | 42 +++--- .../targets_manager_test.go | 49 +++++-- pkg/targetgroupbinding/utils.go | 20 +++ test/framework/framework.go | 3 +- webhooks/elbv2/targetgroupbinding_mutator.go | 28 ++-- .../elbv2/targetgroupbinding_mutator_test.go | 32 ++++- .../elbv2/targetgroupbinding_validator.go | 25 ++-- .../targetgroupbinding_validator_test.go | 11 +- 22 files changed, 480 insertions(+), 140 deletions(-) create mode 100644 pkg/aws/services/cloudInterface.go diff --git a/apis/elbv2/v1beta1/targetgroupbinding_types.go b/apis/elbv2/v1beta1/targetgroupbinding_types.go index 0ec065709b..0ab2b8add5 100644 --- a/apis/elbv2/v1beta1/targetgroupbinding_types.go +++ b/apis/elbv2/v1beta1/targetgroupbinding_types.go @@ -149,6 +149,14 @@ type TargetGroupBindingSpec struct { // VpcID is the VPC of the TargetGroup. If unspecified, it will be automatically inferred. // +optional VpcID string `json:"vpcID,omitempty"` + + // IAM Role ARN to assume when calling AWS APIs. Useful if the target group is in a different AWS account + // +optional + IamRoleArnToAssume string `json:"-"` // `json:"iamRoleArnToAssume,omitempty"` + + // IAM Role ARN to assume when calling AWS APIs. Needed to assume a role in another account and prevent the confused deputy problem. https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html + // +optional + AssumeRoleExternalId string `json:"-"` // `json:"assumeRoleExternalId,omitempty"` } // TargetGroupBindingStatus defines the observed state of TargetGroupBinding diff --git a/controllers/ingress/group_controller.go b/controllers/ingress/group_controller.go index 175bbb6906..8ff259ac60 100644 --- a/controllers/ingress/group_controller.go +++ b/controllers/ingress/group_controller.go @@ -15,7 +15,7 @@ import ( elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/controllers/ingress/eventhandlers" "sigs.k8s.io/aws-load-balancer-controller/pkg/annotations" - "sigs.k8s.io/aws-load-balancer-controller/pkg/aws" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" "sigs.k8s.io/aws-load-balancer-controller/pkg/config" "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy" elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2" @@ -43,7 +43,7 @@ const ( ) // NewGroupReconciler constructs new GroupReconciler -func NewGroupReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, +func NewGroupReconciler(cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, finalizerManager k8s.FinalizerManager, networkingSGManager networkingpkg.SecurityGroupManager, networkingSGReconciler networkingpkg.SecurityGroupReconciler, subnetsResolver networkingpkg.SubnetsResolver, elbv2TaggingManager elbv2deploy.TaggingManager, controllerConfig config.ControllerConfig, backendSGProvider networkingpkg.BackendSGProvider, diff --git a/controllers/service/service_controller.go b/controllers/service/service_controller.go index 2ed7612b01..5b7c87e987 100644 --- a/controllers/service/service_controller.go +++ b/controllers/service/service_controller.go @@ -11,7 +11,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/aws-load-balancer-controller/controllers/service/eventhandlers" "sigs.k8s.io/aws-load-balancer-controller/pkg/annotations" - "sigs.k8s.io/aws-load-balancer-controller/pkg/aws" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" "sigs.k8s.io/aws-load-balancer-controller/pkg/config" "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy" elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2" @@ -34,7 +34,7 @@ const ( controllerName = "service" ) -func NewServiceReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, +func NewServiceReconciler(cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder, finalizerManager k8s.FinalizerManager, networkingSGManager networking.SecurityGroupManager, networkingSGReconciler networking.SecurityGroupReconciler, subnetsResolver networking.SubnetsResolver, vpcInfoProvider networking.VPCInfoProvider, elbv2TaggingManager elbv2deploy.TaggingManager, controllerConfig config.ControllerConfig, diff --git a/docs/deploy/installation.md b/docs/deploy/installation.md index 31fc9af54c..c20742b5be 100644 --- a/docs/deploy/installation.md +++ b/docs/deploy/installation.md @@ -7,10 +7,10 @@ The LBC is supported by AWS. Some clusters may be using the legacy "in-tree" fun !!!question "Existing AWS ALB Ingress Controller users" The AWS ALB Ingress controller must be uninstalled before installing the AWS Load Balancer Controller. Please follow our [migration guide](upgrade/migrate_v1_v2.md) to do a migration. - + !!!warning "When using AWS Load Balancer Controller v2.5+" - The AWS LBC provides a mutating webhook for service resources to set the `spec.loadBalancerClass` field for service of type LoadBalancer on create. - This makes the AWS LBC the **default controller for service** of type LoadBalancer. You can disable this feature and revert to set Cloud Controller Manager (in-tree controller) as the default by setting the helm chart value **enableServiceMutatorWebhook to false** with `--set enableServiceMutatorWebhook=false` . + The AWS LBC provides a mutating webhook for service resources to set the `spec.loadBalancerClass` field for service of type LoadBalancer on create. + This makes the AWS LBC the **default controller for service** of type LoadBalancer. You can disable this feature and revert to set Cloud Controller Manager (in-tree controller) as the default by setting the helm chart value **enableServiceMutatorWebhook to false** with `--set enableServiceMutatorWebhook=false` . You will no longer be able to provision new Classic Load Balancer (CLB) from your kubernetes service unless you disable this feature. Existing CLB will continue to work fine. ## Supported Kubernetes versions @@ -30,7 +30,7 @@ The LBC is supported by AWS. Some clusters may be using the legacy "in-tree" fun Isolated clusters are clusters without internet access, and instead reply on VPC endpoints for all required connects. When installing the AWS LBC in isolated clusters, you need to disable shield, waf and wafv2 via controller flags `--enable-shield=false, --enable-waf=false, --enable-wafv2=false` ### Using the Amazon EC2 instance metadata server version 2 (IMDSv2) -We recommend blocking the access to instance metadata by requiring the instance to use [IMDSv2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html) only. For more information, please refer to the AWS guidance [here](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node). If you are using the IMDSv2, set the hop limit to 2 or higher in order to allow the LBC to perform the metadata introspection. +We recommend blocking the access to instance metadata by requiring the instance to use [IMDSv2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html) only. For more information, please refer to the AWS guidance [here](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node). If you are using the IMDSv2, set the hop limit to 2 or higher in order to allow the LBC to perform the metadata introspection. You can set the IMDSv2 as follows: ``` @@ -127,6 +127,10 @@ If you're not setting up IAM roles for service accounts, apply the IAM policies curl -o iam-policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.7.0/docs/install/iam_policy.json ``` +## Special IAM cases + +### You only want the LBC to add and remove IPs to already existing target groups: + The following IAM permissions subset is for those using `TargetGroupBinding` only and don't plan to use the LBC to manage security group rules: ``` @@ -152,6 +156,57 @@ The following IAM permissions subset is for those using `TargetGroupBinding` onl } ``` +### You only want the LBC to add and remove IPs to already existing target groups, also in other accounts, assuming roles + +On the other hand, if you plan to use the LBC to manage also target groups in different accounts, you will need to add `"sts:AssumeRole"` to your list of permissions, in other words: + +``` +{ + "Statement": [ + { + "Action": [ + "ec2:DescribeVpcs", + "ec2:DescribeSecurityGroups", + "ec2:DescribeInstances", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + "sts:AssumeRole" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" +} +``` + +The assumed roles will need the exactly the same permissions, without `"sts:AssumeRole"`. The assumed role will need a to allow to be assumed by the main role, something like this: + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::999999999999999:user/test-alb-controller" + }, + "Action": "sts:AssumeRole", + "Condition": { + "StringEquals": { + "sts:ExternalId": "very-secret-string" + } + } + } + ] +} +``` + ## Network configuration Review the [worker nodes security group](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) docs. Your node security group must permit incoming traffic on TCP port 9443 from the Kubernetes control plane. This is needed for webhook access. diff --git a/docs/guide/targetgroupbinding/spec.md b/docs/guide/targetgroupbinding/spec.md index de865b5304..f2b9b80c55 100644 --- a/docs/guide/targetgroupbinding/spec.md +++ b/docs/guide/targetgroupbinding/spec.md @@ -52,10 +52,29 @@ Kubernetes meta/v1.ObjectMeta -Refer to the Kubernetes API documentation for the fields of the + + +
annotations + + + + + + +
alb.ingress.kubernetes.io/IamRoleArnToAssume
string		(Optional) In case the target group is in a differet AWS account, you put here the role that needs to be assumed in order to manipulate the target group. +
alb.ingress.kubernetes.io/AssumeRoleExternalId
string		(Optional) The external ID for the assume role operation. Optional, but recommended. It helps you to prevent the confused deputy problem. +
+ +
+Refer to the Kubernetes API documentation for the other fields of the metadata field. +
+ + + + spec
diff --git a/docs/guide/targetgroupbinding/targetgroupbinding.md b/docs/guide/targetgroupbinding/targetgroupbinding.md index 36cdf065d3..25d0dfe81a 100644 --- a/docs/guide/targetgroupbinding/targetgroupbinding.md +++ b/docs/guide/targetgroupbinding/targetgroupbinding.md @@ -92,6 +92,28 @@ spec: ... ``` +### AssumeRole + +Sometimes the AWS LoadBalancer controller needs to manipulate target groups from different AWS accounts. +The way to do that is assuming a role from such account. There are annotations that can help you with that: + +* `alb.ingress.kubernetes.io/IamRoleArnToAssume`: the ARN that you need to assume +* `alb.ingress.kubernetes.io/AssumeRoleExternalId`: the external ID for the assume role operation. Optional, but recommended. It helps you to prevent the confused deputy problem ( https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html ) + +```yaml +apiVersion: elbv2.k8s.aws/v1beta1 +kind: TargetGroupBinding +metadata: + name: my-tgb + annotations: + alb.ingress.kubernetes.io/IamRoleArnToAssume: "arn:aws:iam::999999999999:role/alb-controller-policy-to-assume" + alb.ingress.kubernetes.io/AssumeRoleExternalId: "some-magic-string" +spec: + ... +``` + + + ## Reference See the [reference](./spec.md) for TargetGroupBinding CR diff --git a/go.mod b/go.mod index 2eeae10d62..f858b23672 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module sigs.k8s.io/aws-load-balancer-controller go 1.22.7 require ( - github.com/aws/aws-sdk-go-v2 v1.30.5 + github.com/aws/aws-sdk-go-v2 v1.31.0 github.com/aws/aws-sdk-go-v2/config v1.27.27 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 github.com/aws/aws-sdk-go-v2/service/acm v1.28.4 @@ -55,10 +55,12 @@ require ( github.com/ajg/form v1.5.1 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect + github.com/aws/aws-sdk-go v1.55.5 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/iam v1.36.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect diff --git a/go.sum b/go.sum index bfb19e52e9..4b709832b2 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,12 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g= github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= +github.com/aws/aws-sdk-go-v2 v1.31.0 h1:3V05LbxTSItI5kUqNwhJrrrY1BAXxXt0sN0l72QmG5U= +github.com/aws/aws-sdk-go-v2 v1.31.0/go.mod h1:ztolYtaEUtdpf9Wftr31CJfLVjOnD/CVRkKOOYgF8hA= github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90= github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg= github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI= @@ -46,8 +50,12 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 h1:kYQ3H1u0ANr9KEKlGs/jTLrBFPo8P8NaH/w7A01NeeM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18/go.mod h1:r506HmK5JDUh9+Mw4CfGJGSSoqIiLCndAuqXuhbv67Y= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 h1:Z7IdFUONvTcvS7YuhtVxN99v2cCoHRXOS4mTr0B/pUc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18/go.mod h1:DkKMmksZVVyat+Y+r1dEOgJEfUeA7UngIHWeKsi0yNc= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/service/acm v1.28.4 h1:wiW1Y6/1lysA0eJZRq0I53YYKuV9MNAzL15z2eZRlEE= @@ -58,6 +66,8 @@ github.com/aws/aws-sdk-go-v2/service/ec2 v1.173.0 h1:ta62lid9JkIpKZtZZXSj6rP2AqY github.com/aws/aws-sdk-go-v2/service/ec2 v1.173.0/go.mod h1:o6QDjdVKpP5EF0dp/VlvqckzuSDATr1rLdHt3A5m0YY= github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.36.0 h1:3t8g6wmPA9hr69qzDraI1umO2An7jKNe75dBsxbI30E= github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.36.0/go.mod h1:jk+iid9R4MN7UVDwSTK/ZDDO8WNhxnO2WVzfYOMLh+4= +github.com/aws/aws-sdk-go-v2/service/iam v1.36.3 h1:dV9iimLEHKYAz2qTi+tGAD9QCnAG2pLD7HUEHB7m4mI= +github.com/aws/aws-sdk-go-v2/service/iam v1.36.3/go.mod h1:HSvujsK8xeEHMIB18oMXjSfqaN9cVqpo/MtHJIksQRk= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= diff --git a/pkg/aws/cloud.go b/pkg/aws/cloud.go index 1ebe084dab..4a4ca92397 100644 --- a/pkg/aws/cloud.go +++ b/pkg/aws/cloud.go @@ -3,18 +3,24 @@ package aws import ( "context" "fmt" + "log" + "net" + "os" + "strings" + awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/aws-sdk-go-v2/aws/ratelimit" "github.com/aws/aws-sdk-go-v2/aws/retry" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/sts" + smithymiddleware "github.com/aws/smithy-go/middleware" - "net" - "os" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/endpoints" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/metrics" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/throttle" "sigs.k8s.io/aws-load-balancer-controller/pkg/version" - "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" @@ -29,37 +35,8 @@ import ( const userAgent = "elbv2.k8s.aws" -type Cloud interface { - // EC2 provides API to AWS EC2 - EC2() services.EC2 - - // ELBV2 provides API to AWS ELBV2 - ELBV2() services.ELBV2 - - // ACM provides API to AWS ACM - ACM() services.ACM - - // WAFv2 provides API to AWS WAFv2 - WAFv2() services.WAFv2 - - // WAFRegional provides API to AWS WAFRegional - WAFRegional() services.WAFRegional - - // Shield provides API to AWS Shield - Shield() services.Shield - - // RGT provides API to AWS RGT - RGT() services.RGT - - // Region for the kubernetes cluster - Region() string - - // VpcID for the LoadBalancer resources. - VpcID() string -} - // NewCloud constructs new Cloud implementation. -func NewCloud(cfg CloudConfig, metricsRegisterer prometheus.Registerer, logger logr.Logger) (Cloud, error) { +func NewCloud(cfg CloudConfig, metricsRegisterer prometheus.Registerer, logger logr.Logger) (services.Cloud, error) { hasIPv4 := true addrs, err := net.InterfaceAddrs() if err == nil { @@ -135,17 +112,29 @@ func NewCloud(cfg CloudConfig, metricsRegisterer prometheus.Registerer, logger l if err != nil { return nil, errors.Wrap(err, "failed to get VPC ID") } + cfg.VpcID = vpcID - return &defaultCloud{ - cfg: cfg, - ec2: ec2Service, - elbv2: services.NewELBV2(awsConfig, endpointsResolver), + + thisObj := &defaultCloud{ + cfg: cfg, + ec2: ec2Service, + // elbv2: services.NewELBV2(awsConfig, endpointsResolver), acm: services.NewACM(awsConfig, endpointsResolver), wafv2: services.NewWAFv2(awsConfig, endpointsResolver), wafRegional: services.NewWAFRegional(awsConfig, endpointsResolver, cfg.Region), shield: services.NewShield(awsConfig, endpointsResolver), //done rgt: services.NewRGT(awsConfig, endpointsResolver), - }, nil + + assumeRoleElbV2: make(map[string]services.ELBV2), + // session: sess, + endpointsResolver: endpointsResolver, + awsConfig: &awsConfig, + logger: logger, + } + + thisObj.elbv2 = services.NewELBV2(awsConfig, endpointsResolver, thisObj) + + return thisObj, nil } func getVpcID(cfg CloudConfig, ec2Service services.EC2, ec2Metadata services.EC2Metadata, logger logr.Logger) (string, error) { @@ -220,7 +209,7 @@ func inferVPCIDFromTags(ec2Service services.EC2, VpcNameTagKey string, VpcNameTa return *vpcs[0].VpcId, nil } -var _ Cloud = &defaultCloud{} +var _ services.Cloud = &defaultCloud{} type defaultCloud struct { cfg CloudConfig @@ -232,6 +221,75 @@ type defaultCloud struct { wafRegional services.WAFRegional shield services.Shield rgt services.RGT + + assumeRoleElbV2 map[string]services.ELBV2 + endpointsResolver *endpoints.Resolver + awsConfig *aws.Config + logger logr.Logger +} + +// returns ELBV2 client for the given assumeRoleArn, or the default ELBV2 client if assumeRoleArn is empty +func (c *defaultCloud) GetAssumedRoleELBV2(ctx context.Context, assumeRoleArn string, externalId string) services.ELBV2 { + + if assumeRoleArn == "" { + return c.elbv2 + } + + assumedRoleELBV2, exists := c.assumeRoleElbV2[assumeRoleArn] + if exists { + return assumedRoleELBV2 + } + c.logger.Info("awsCloud", "method", "GetAssumedRoleELBV2", "AssumeRoleArn", assumeRoleArn, "externalId", externalId) + + //////////////// + sourceAccount := sts.NewFromConfig(*c.awsConfig) + response, err := sourceAccount.AssumeRole(ctx, &sts.AssumeRoleInput{ + RoleArn: aws.String(assumeRoleArn), + RoleSessionName: aws.String("aws-load-balancer-controller"), + ExternalId: aws.String(externalId), + }) + if err != nil { + log.Fatalf("Unable to assume target role, %v. Attempting to use default client", err) + return c.elbv2 + } + assumedRoleCreds := response.Credentials + newCreds := credentials.NewStaticCredentialsProvider(*assumedRoleCreds.AccessKeyId, *assumedRoleCreds.SecretAccessKey, *assumedRoleCreds.SessionToken) + newAwsConfig, err := config.LoadDefaultConfig(ctx, config.WithRegion(c.cfg.Region), config.WithCredentialsProvider(newCreds)) + if err != nil { + log.Fatalf("Unable to load static credentials for service client config, %v. Attempting to use default client", err) + return c.elbv2 + } + + c.awsConfig.Credentials = newAwsConfig.Credentials // response.Credentials + + // // var assumedRoleCreds *stsTypes.Credentials = response.Credentials + + // // Create config with target service client, using assumed role + // cfg, err = config.LoadDefaultConfig(ctx, config.WithRegion(region), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(*assumedRoleCreds.AccessKeyId, *assumedRoleCreds.SecretAccessKey, *assumedRoleCreds.SessionToken))) + // if err != nil { + // log.Fatalf("unable to load static credentials for service client config, %v", err) + // } + + // //////////////// + // appCreds := stscreds.NewAssumeRoleProvider(client, assumeRoleArn) + // value, err := appCreds.Retrieve(context.TODO()) + // if err != nil { + // // handle error + // } + // ///////// + + // ///////////// OLD + // creds := stscreds.NewCredentials(c.session, assumeRoleArn, func(p *stscreds.AssumeRoleProvider) { + // p.ExternalID = &externalId + // }) + // ////////////// + + // c.awsConfig.Credentials = creds + // // newObj := services.NewELBV2(c.session, c, c.awsCFG) + newObj := services.NewELBV2(*c.awsConfig, c.endpointsResolver, c) + c.assumeRoleElbV2[assumeRoleArn] = newObj + + return newObj } func (c *defaultCloud) EC2() services.EC2 { diff --git a/pkg/aws/services/cloudInterface.go b/pkg/aws/services/cloudInterface.go new file mode 100644 index 0000000000..3e0ff558ec --- /dev/null +++ b/pkg/aws/services/cloudInterface.go @@ -0,0 +1,34 @@ +package services + +import "context" + +type Cloud interface { + // EC2 provides API to AWS EC2 + EC2() EC2 + + // ELBV2 provides API to AWS ELBV2 + ELBV2() ELBV2 + + // ACM provides API to AWS ACM + ACM() ACM + + // WAFv2 provides API to AWS WAFv2 + WAFv2() WAFv2 + + // WAFRegional provides API to AWS WAFRegional + WAFRegional() WAFRegional + + // Shield provides API to AWS Shield + Shield() Shield + + // RGT provides API to AWS RGT + RGT() RGT + + // Region for the kubernetes cluster + Region() string + + // VpcID for the LoadBalancer resources. + VpcID() string + + GetAssumedRoleELBV2(ctx context.Context, assumeRoleArn string, externalId string) ELBV2 +} diff --git a/pkg/aws/services/elbv2.go b/pkg/aws/services/elbv2.go index 0ff0e7d187..239980bd7c 100644 --- a/pkg/aws/services/elbv2.go +++ b/pkg/aws/services/elbv2.go @@ -25,7 +25,6 @@ type ELBV2 interface { // wrapper to DescribeRulesWithContext API, which aggregates paged results into list. DescribeRulesAsList(ctx context.Context, input *elasticloadbalancingv2.DescribeRulesInput) ([]types.Rule, error) - AddTagsWithContext(ctx context.Context, input *elasticloadbalancingv2.AddTagsInput) (*elasticloadbalancingv2.AddTagsOutput, error) RemoveTagsWithContext(ctx context.Context, input *elasticloadbalancingv2.RemoveTagsInput) (*elasticloadbalancingv2.RemoveTagsOutput, error) DescribeTagsWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeTagsInput) (*elasticloadbalancingv2.DescribeTagsOutput, error) @@ -60,21 +59,30 @@ type ELBV2 interface { AddListenerCertificatesWithContext(ctx context.Context, input *elasticloadbalancingv2.AddListenerCertificatesInput) (*elasticloadbalancingv2.AddListenerCertificatesOutput, error) DescribeListenerAttributesWithContext(ctx context.Context, input *elasticloadbalancingv2.DescribeListenerAttributesInput) (*elasticloadbalancingv2.DescribeListenerAttributesOutput, error) ModifyListenerAttributesWithContext(ctx context.Context, input *elasticloadbalancingv2.ModifyListenerAttributesInput) (*elasticloadbalancingv2.ModifyListenerAttributesOutput, error) + AssumeRole(ctx context.Context, assumeRoleArn string, externalId string) ELBV2 } -func NewELBV2(cfg aws.Config, endpointsResolver *endpoints.Resolver) ELBV2 { +func NewELBV2(cfg aws.Config, endpointsResolver *endpoints.Resolver, cloud Cloud) ELBV2 { customEndpoint := endpointsResolver.EndpointFor(elasticloadbalancingv2.ServiceID) client := elasticloadbalancingv2.NewFromConfig(cfg, func(o *elasticloadbalancingv2.Options) { if customEndpoint != nil { o.BaseEndpoint = customEndpoint } }) - return &elbv2Client{elbv2Client: client} + return &elbv2Client{elbv2Client: client, cloud: cloud} } // default implementation for ELBV2. type elbv2Client struct { elbv2Client *elasticloadbalancingv2.Client + cloud Cloud +} + +func (c *elbv2Client) AssumeRole(ctx context.Context, assumeRoleArn string, externalId string) ELBV2 { + if assumeRoleArn == "" { + return c + } + return c.cloud.GetAssumedRoleELBV2(ctx, assumeRoleArn, externalId) } func (c *elbv2Client) AddListenerCertificatesWithContext(ctx context.Context, input *elasticloadbalancingv2.AddListenerCertificatesInput) (*elasticloadbalancingv2.AddListenerCertificatesOutput, error) { diff --git a/pkg/aws/services/elbv2_mocks.go b/pkg/aws/services/elbv2_mocks.go index 42c82464c6..9aaa3d3227 100644 --- a/pkg/aws/services/elbv2_mocks.go +++ b/pkg/aws/services/elbv2_mocks.go @@ -66,6 +66,20 @@ func (mr *MockELBV2MockRecorder) AddTagsWithContext(arg0, arg1 interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddTagsWithContext", reflect.TypeOf((*MockELBV2)(nil).AddTagsWithContext), arg0, arg1) } +// AssumeRole mocks base method. +func (m *MockELBV2) AssumeRole(arg0 context.Context, arg1, arg2 string) ELBV2 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AssumeRole", arg0, arg1, arg2) + ret0, _ := ret[0].(ELBV2) + return ret0 +} + +// AssumeRole indicates an expected call of AssumeRole. +func (mr *MockELBV2MockRecorder) AssumeRole(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AssumeRole", reflect.TypeOf((*MockELBV2)(nil).AssumeRole), arg0, arg1, arg2) +} + // CreateListenerWithContext mocks base method. func (m *MockELBV2) CreateListenerWithContext(arg0 context.Context, arg1 *elasticloadbalancingv2.CreateListenerInput) (*elasticloadbalancingv2.CreateListenerOutput, error) { m.ctrl.T.Helper() diff --git a/pkg/deploy/stack_deployer.go b/pkg/deploy/stack_deployer.go index dda035adc3..8456500ed1 100644 --- a/pkg/deploy/stack_deployer.go +++ b/pkg/deploy/stack_deployer.go @@ -2,8 +2,9 @@ package deploy import ( "context" + "github.com/go-logr/logr" - "sigs.k8s.io/aws-load-balancer-controller/pkg/aws" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" "sigs.k8s.io/aws-load-balancer-controller/pkg/config" "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/ec2" "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2" @@ -23,7 +24,7 @@ type StackDeployer interface { } // NewDefaultStackDeployer constructs new defaultStackDeployer. -func NewDefaultStackDeployer(cloud aws.Cloud, k8sClient client.Client, +func NewDefaultStackDeployer(cloud services.Cloud, k8sClient client.Client, networkingSGManager networking.SecurityGroupManager, networkingSGReconciler networking.SecurityGroupReconciler, elbv2TaggingManager elbv2.TaggingManager, config config.ControllerConfig, tagPrefix string, logger logr.Logger) *defaultStackDeployer { @@ -57,7 +58,7 @@ var _ StackDeployer = &defaultStackDeployer{} // defaultStackDeployer is the default implementation for StackDeployer type defaultStackDeployer struct { - cloud aws.Cloud + cloud services.Cloud k8sClient client.Client addonsConfig config.AddonsConfig trackingProvider tracking.Provider diff --git a/pkg/targetgroupbinding/resource_manager.go b/pkg/targetgroupbinding/resource_manager.go index d9b263ea28..2f72bf292c 100644 --- a/pkg/targetgroupbinding/resource_manager.go +++ b/pkg/targetgroupbinding/resource_manager.go @@ -3,11 +3,12 @@ package targetgroupbinding import ( "context" "fmt" - elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "github.com/aws/smithy-go" "net/netip" "time" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + "github.com/aws/smithy-go" + "k8s.io/client-go/tools/record" awssdk "github.com/aws/aws-sdk-go-v2/aws" @@ -85,13 +86,16 @@ func (m *defaultResourceManager) Reconcile(ctx context.Context, tgb *elbv2api.Ta if tgb.Spec.TargetType == nil { return errors.Errorf("targetType is not specified: %v", k8s.NamespacedName(tgb).String()) } + AnnotationsToFields(tgb) if *tgb.Spec.TargetType == elbv2api.TargetTypeIP { + return m.reconcileWithIPTargetType(ctx, tgb) } return m.reconcileWithInstanceTargetType(ctx, tgb) } func (m *defaultResourceManager) Cleanup(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error { + AnnotationsToFields(tgb) if err := m.cleanupTargets(ctx, tgb); err != nil { return err } @@ -126,9 +130,7 @@ func (m *defaultResourceManager) reconcileWithIPTargetType(ctx context.Context, return err } - tgARN := tgb.Spec.TargetGroupARN - vpcID := tgb.Spec.VpcID - targets, err := m.targetsManager.ListTargets(ctx, tgARN) + targets, err := m.targetsManager.ListTargets(ctx, tgb) if err != nil { return err } @@ -141,12 +143,12 @@ func (m *defaultResourceManager) reconcileWithIPTargetType(ctx context.Context, needNetworkingRequeue = true } if len(unmatchedTargets) > 0 { - if err := m.deregisterTargets(ctx, tgARN, unmatchedTargets); err != nil { + if err := m.deregisterTargets(ctx, tgb, unmatchedTargets); err != nil { return err } } if len(unmatchedEndpoints) > 0 { - if err := m.registerPodEndpoints(ctx, tgARN, vpcID, unmatchedEndpoints); err != nil { + if err := m.registerPodEndpoints(ctx, tgb, unmatchedEndpoints); err != nil { return err } } @@ -191,8 +193,7 @@ func (m *defaultResourceManager) reconcileWithInstanceTargetType(ctx context.Con } return err } - tgARN := tgb.Spec.TargetGroupARN - targets, err := m.targetsManager.ListTargets(ctx, tgARN) + targets, err := m.targetsManager.ListTargets(ctx, tgb) if err != nil { return err } @@ -203,12 +204,12 @@ func (m *defaultResourceManager) reconcileWithInstanceTargetType(ctx context.Con return err } if len(unmatchedTargets) > 0 { - if err := m.deregisterTargets(ctx, tgARN, unmatchedTargets); err != nil { + if err := m.deregisterTargets(ctx, tgb, unmatchedTargets); err != nil { return err } } if len(unmatchedEndpoints) > 0 { - if err := m.registerNodePortEndpoints(ctx, tgARN, unmatchedEndpoints); err != nil { + if err := m.registerNodePortEndpoints(ctx, tgb, unmatchedEndpoints); err != nil { return err } } @@ -217,7 +218,7 @@ func (m *defaultResourceManager) reconcileWithInstanceTargetType(ctx context.Con } func (m *defaultResourceManager) cleanupTargets(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error { - targets, err := m.targetsManager.ListTargets(ctx, tgb.Spec.TargetGroupARN) + targets, err := m.targetsManager.ListTargets(ctx, tgb) if err != nil { if isELBV2TargetGroupNotFoundError(err) { return nil @@ -226,7 +227,7 @@ func (m *defaultResourceManager) cleanupTargets(ctx context.Context, tgb *elbv2a } return err } - if err := m.deregisterTargets(ctx, tgb.Spec.TargetGroupARN, targets); err != nil { + if err := m.deregisterTargets(ctx, tgb, targets); err != nil { if isELBV2TargetGroupNotFoundError(err) { return nil } else if isELBV2TargetGroupARNInvalidError(err) { @@ -375,21 +376,39 @@ func (m *defaultResourceManager) updatePodAsHealthyForDeletedTGB(ctx context.Con return nil } -func (m *defaultResourceManager) deregisterTargets(ctx context.Context, tgARN string, targets []TargetInfo) error { +func (m *defaultResourceManager) deregisterTargets(ctx context.Context, tgb *elbv2api.TargetGroupBinding, targets []TargetInfo) error { sdkTargets := make([]elbv2types.TargetDescription, 0, len(targets)) for _, target := range targets { sdkTargets = append(sdkTargets, target.Target) } - return m.targetsManager.DeregisterTargets(ctx, tgARN, sdkTargets) + return m.targetsManager.DeregisterTargets(ctx, tgb, sdkTargets) } -func (m *defaultResourceManager) registerPodEndpoints(ctx context.Context, tgARN, tgVpcID string, endpoints []backend.PodEndpoint) error { +func (m *defaultResourceManager) registerPodEndpoints(ctx context.Context, tgb *elbv2api.TargetGroupBinding, endpoints []backend.PodEndpoint) error { vpcID := m.vpcID // Target group is in a different VPC from the cluster's VPC - if tgVpcID != "" && tgVpcID != m.vpcID { - vpcID = tgVpcID - m.logger.Info("registering endpoints using the targetGroup's vpcID", tgVpcID, - "which is different from the cluster's vpcID", m.vpcID) + if tgb.Spec.VpcID != "" && tgb.Spec.VpcID != m.vpcID { + vpcID = tgb.Spec.VpcID + m.logger.Info(fmt.Sprintf( + "registering endpoints using the targetGroup's vpcID %s which is different from the cluster's vpcID %s", tgb.Spec.VpcID, m.vpcID)) + + if tgb.Spec.IamRoleArnToAssume != "" { + // since we need to assume a role for this TGB, + // it is from a different account + // so the packets will need to leave the VPC and therefore + // target.AvailabilityZone = awssdk.String("all") must be set + // or else nothing will work + sdkTargets := make([]elbv2types.TargetDescription, 0, len(endpoints)) + for _, endpoint := range endpoints { + target := elbv2types.TargetDescription{ + Id: awssdk.String(endpoint.IP), + Port: awssdk.Int32(endpoint.Port), + } + target.AvailabilityZone = awssdk.String("all") + sdkTargets = append(sdkTargets, target) + } + return m.targetsManager.RegisterTargets(ctx, tgb, sdkTargets) + } } vpcInfo, err := m.vpcInfoProvider.FetchVPCInfo(ctx, vpcID) if err != nil { @@ -418,10 +437,10 @@ func (m *defaultResourceManager) registerPodEndpoints(ctx context.Context, tgARN } sdkTargets = append(sdkTargets, target) } - return m.targetsManager.RegisterTargets(ctx, tgARN, sdkTargets) + return m.targetsManager.RegisterTargets(ctx, tgb, sdkTargets) } -func (m *defaultResourceManager) registerNodePortEndpoints(ctx context.Context, tgARN string, endpoints []backend.NodePortEndpoint) error { +func (m *defaultResourceManager) registerNodePortEndpoints(ctx context.Context, tgb *elbv2api.TargetGroupBinding, endpoints []backend.NodePortEndpoint) error { sdkTargets := make([]elbv2types.TargetDescription, 0, len(endpoints)) for _, endpoint := range endpoints { sdkTargets = append(sdkTargets, elbv2types.TargetDescription{ @@ -429,7 +448,7 @@ func (m *defaultResourceManager) registerNodePortEndpoints(ctx context.Context, Port: awssdk.Int32(endpoint.Port), }) } - return m.targetsManager.RegisterTargets(ctx, tgARN, sdkTargets) + return m.targetsManager.RegisterTargets(ctx, tgb, sdkTargets) } type podEndpointAndTargetPair struct { diff --git a/pkg/targetgroupbinding/targets_manager.go b/pkg/targetgroupbinding/targets_manager.go index b32eb350e0..b00f95a9b5 100644 --- a/pkg/targetgroupbinding/targets_manager.go +++ b/pkg/targetgroupbinding/targets_manager.go @@ -2,14 +2,16 @@ package targetgroupbinding import ( "context" + "sync" + "time" + "github.com/aws/aws-sdk-go-v2/aws" elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/cache" + elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" - "sync" - "time" ) const ( @@ -21,13 +23,13 @@ const ( // TargetsManager is an abstraction around ELBV2's targets API. type TargetsManager interface { // Register Targets into TargetGroup. - RegisterTargets(ctx context.Context, tgARN string, targets []elbv2types.TargetDescription) error + RegisterTargets(ctx context.Context, tgb *elbv2api.TargetGroupBinding, targets []elbv2types.TargetDescription) error // Deregister Targets from TargetGroup. - DeregisterTargets(ctx context.Context, tgARN string, targets []elbv2types.TargetDescription) error + DeregisterTargets(ctx context.Context, tgb *elbv2api.TargetGroupBinding, targets []elbv2types.TargetDescription) error // List Targets from TargetGroup. - ListTargets(ctx context.Context, tgARN string) ([]TargetInfo, error) + ListTargets(ctx context.Context, tgb *elbv2api.TargetGroupBinding) ([]TargetInfo, error) } // NewCachedTargetsManager constructs new cachedTargetsManager @@ -76,7 +78,8 @@ type targetsCacheItem struct { targets []TargetInfo } -func (m *cachedTargetsManager) RegisterTargets(ctx context.Context, tgARN string, targets []elbv2types.TargetDescription) error { +func (m *cachedTargetsManager) RegisterTargets(ctx context.Context, tgb *elbv2api.TargetGroupBinding, targets []elbv2types.TargetDescription) error { + tgARN := tgb.Spec.TargetGroupARN targetsChunks := chunkTargetDescriptions(targets, m.registerTargetsChunkSize) for _, targetsChunk := range targetsChunks { req := &elbv2sdk.RegisterTargetsInput{ @@ -86,7 +89,7 @@ func (m *cachedTargetsManager) RegisterTargets(ctx context.Context, tgARN string m.logger.Info("registering targets", "arn", tgARN, "targets", targetsChunk) - _, err := m.elbv2Client.RegisterTargetsWithContext(ctx, req) + _, err := m.elbv2Client.AssumeRole(ctx, tgb.Spec.IamRoleArnToAssume, tgb.Spec.AssumeRoleExternalId).RegisterTargetsWithContext(ctx, req) if err != nil { return err } @@ -97,7 +100,8 @@ func (m *cachedTargetsManager) RegisterTargets(ctx context.Context, tgARN string return nil } -func (m *cachedTargetsManager) DeregisterTargets(ctx context.Context, tgARN string, targets []elbv2types.TargetDescription) error { +func (m *cachedTargetsManager) DeregisterTargets(ctx context.Context, tgb *elbv2api.TargetGroupBinding, targets []elbv2types.TargetDescription) error { + tgARN := tgb.Spec.TargetGroupARN targetsChunks := chunkTargetDescriptions(targets, m.deregisterTargetsChunkSize) for _, targetsChunk := range targetsChunks { req := &elbv2sdk.DeregisterTargetsInput{ @@ -107,7 +111,7 @@ func (m *cachedTargetsManager) DeregisterTargets(ctx context.Context, tgARN stri m.logger.Info("deRegistering targets", "arn", tgARN, "targets", targetsChunk) - _, err := m.elbv2Client.DeregisterTargetsWithContext(ctx, req) + _, err := m.elbv2Client.AssumeRole(ctx, tgb.Spec.IamRoleArnToAssume, tgb.Spec.AssumeRoleExternalId).DeregisterTargetsWithContext(ctx, req) if err != nil { return err } @@ -118,7 +122,8 @@ func (m *cachedTargetsManager) DeregisterTargets(ctx context.Context, tgARN stri return nil } -func (m *cachedTargetsManager) ListTargets(ctx context.Context, tgARN string) ([]TargetInfo, error) { +func (m *cachedTargetsManager) ListTargets(ctx context.Context, tgb *elbv2api.TargetGroupBinding) ([]TargetInfo, error) { + tgARN := tgb.Spec.TargetGroupARN m.targetsCacheMutex.Lock() defer m.targetsCacheMutex.Unlock() @@ -126,7 +131,7 @@ func (m *cachedTargetsManager) ListTargets(ctx context.Context, tgARN string) ([ targetsCacheItem := rawTargetsCacheItem.(*targetsCacheItem) targetsCacheItem.mutex.Lock() defer targetsCacheItem.mutex.Unlock() - refreshedTargets, err := m.refreshUnhealthyTargets(ctx, tgARN, targetsCacheItem.targets) + refreshedTargets, err := m.refreshUnhealthyTargets(ctx, tgb, targetsCacheItem.targets) if err != nil { return nil, err } @@ -134,7 +139,7 @@ func (m *cachedTargetsManager) ListTargets(ctx context.Context, tgARN string) ([ return cloneTargetInfoSlice(refreshedTargets), nil } - refreshedTargets, err := m.refreshAllTargets(ctx, tgARN) + refreshedTargets, err := m.refreshAllTargets(ctx, tgb) if err != nil { return nil, err } @@ -147,8 +152,8 @@ func (m *cachedTargetsManager) ListTargets(ctx context.Context, tgARN string) ([ } // refreshAllTargets will refresh all targets for targetGroup. -func (m *cachedTargetsManager) refreshAllTargets(ctx context.Context, tgARN string) ([]TargetInfo, error) { - targets, err := m.listTargetsFromAWS(ctx, tgARN, nil) +func (m *cachedTargetsManager) refreshAllTargets(ctx context.Context, tgb *elbv2api.TargetGroupBinding) ([]TargetInfo, error) { + targets, err := m.listTargetsFromAWS(ctx, tgb, nil) if err != nil { return nil, err } @@ -158,7 +163,7 @@ func (m *cachedTargetsManager) refreshAllTargets(ctx context.Context, tgARN stri // refreshUnhealthyTargets will refresh targets that are not in healthy status for targetGroup. // To save API calls, we don't refresh targets that are already healthy since once a target turns healthy, we'll unblock it's readinessProbe. // we can do nothing from controller perspective when a healthy target becomes unhealthy. -func (m *cachedTargetsManager) refreshUnhealthyTargets(ctx context.Context, tgARN string, cachedTargets []TargetInfo) ([]TargetInfo, error) { +func (m *cachedTargetsManager) refreshUnhealthyTargets(ctx context.Context, tgb *elbv2api.TargetGroupBinding, cachedTargets []TargetInfo) ([]TargetInfo, error) { var refreshedTargets []TargetInfo var unhealthyTargets []elbv2types.TargetDescription for _, cachedTarget := range cachedTargets { @@ -172,7 +177,7 @@ func (m *cachedTargetsManager) refreshUnhealthyTargets(ctx context.Context, tgAR return refreshedTargets, nil } - refreshedUnhealthyTargets, err := m.listTargetsFromAWS(ctx, tgARN, unhealthyTargets) + refreshedUnhealthyTargets, err := m.listTargetsFromAWS(ctx, tgb, unhealthyTargets) if err != nil { return nil, err } @@ -188,12 +193,13 @@ func (m *cachedTargetsManager) refreshUnhealthyTargets(ctx context.Context, tgAR // listTargetsFromAWS will list targets for TargetGroup using ELBV2API. // if specified targets is non-empty, only these targets will be listed. // otherwise, all targets for targetGroup will be listed. -func (m *cachedTargetsManager) listTargetsFromAWS(ctx context.Context, tgARN string, targets []elbv2types.TargetDescription) ([]TargetInfo, error) { +func (m *cachedTargetsManager) listTargetsFromAWS(ctx context.Context, tgb *elbv2api.TargetGroupBinding, targets []elbv2types.TargetDescription) ([]TargetInfo, error) { + tgARN := tgb.Spec.TargetGroupARN req := &elbv2sdk.DescribeTargetHealthInput{ TargetGroupArn: aws.String(tgARN), Targets: pointerizeTargetDescriptions(targets), } - resp, err := m.elbv2Client.DescribeTargetHealthWithContext(ctx, req) + resp, err := m.elbv2Client.AssumeRole(ctx, tgb.Spec.IamRoleArnToAssume, tgb.Spec.AssumeRoleExternalId).DescribeTargetHealthWithContext(ctx, req) if err != nil { return nil, err } diff --git a/pkg/targetgroupbinding/targets_manager_test.go b/pkg/targetgroupbinding/targets_manager_test.go index 1a291ffd1e..f4f476fe03 100644 --- a/pkg/targetgroupbinding/targets_manager_test.go +++ b/pkg/targetgroupbinding/targets_manager_test.go @@ -2,19 +2,34 @@ package targetgroupbinding import ( "context" + "sync" + "testing" + "time" + awssdk "github.com/aws/aws-sdk-go-v2/aws" elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + + "github.com/golang/mock/gomock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/cache" + elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" "sigs.k8s.io/controller-runtime/pkg/log" - "sync" - "testing" - "time" ) +func makeTargetGroupBinding(tgARN string) *elbv2api.TargetGroupBinding { + return &elbv2api.TargetGroupBinding{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + Spec: elbv2api.TargetGroupBindingSpec{ + TargetGroupARN: tgARN, + }, + } +} + func Test_cachedTargetsManager_RegisterTargets(t *testing.T) { type registerTargetsWithContextCall struct { req *elbv2sdk.RegisterTargetsInput @@ -262,8 +277,10 @@ func Test_cachedTargetsManager_RegisterTargets(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() elbv2Client := services.NewMockELBV2(ctrl) + ctx := context.Background() for _, call := range tt.fields.registerTargetsWithContextCalls { elbv2Client.EXPECT().RegisterTargetsWithContext(gomock.Any(), call.req).Return(call.resp, call.err) + elbv2Client.EXPECT().AssumeRole(ctx, gomock.Any(), gomock.Any()).Return(elbv2Client) } targetsCache := cache.NewExpiring() @@ -282,8 +299,7 @@ func Test_cachedTargetsManager_RegisterTargets(t *testing.T) { logger: log.Log, } - ctx := context.Background() - err := m.RegisterTargets(ctx, tt.args.tgARN, tt.args.targets) + err := m.RegisterTargets(ctx, makeTargetGroupBinding(tt.args.tgARN), tt.args.targets) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) } else { @@ -507,8 +523,10 @@ func Test_cachedTargetsManager_DeregisterTargets(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() elbv2Client := services.NewMockELBV2(ctrl) + ctx := context.Background() for _, call := range tt.fields.deregisterTargetsWithContextCalls { elbv2Client.EXPECT().DeregisterTargetsWithContext(gomock.Any(), call.req).Return(call.resp, call.err) + elbv2Client.EXPECT().AssumeRole(ctx, gomock.Any(), gomock.Any()).Return(elbv2Client) } targetsCache := cache.NewExpiring() @@ -527,8 +545,7 @@ func Test_cachedTargetsManager_DeregisterTargets(t *testing.T) { logger: log.Log, } - ctx := context.Background() - err := m.DeregisterTargets(ctx, tt.args.tgARN, tt.args.targets) + err := m.DeregisterTargets(ctx, makeTargetGroupBinding(tt.args.tgARN), tt.args.targets) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) } else { @@ -770,10 +787,12 @@ func Test_cachedTargetsManager_ListTargets(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() + ctx := context.Background() elbv2Client := services.NewMockELBV2(ctrl) for _, call := range tt.fields.describeTargetHealthWithContextCalls { elbv2Client.EXPECT().DescribeTargetHealthWithContext(gomock.Any(), call.req).Return(call.resp, call.err) + elbv2Client.EXPECT().AssumeRole(ctx, gomock.Any(), gomock.Any()).Return(elbv2Client) } targetsCache := cache.NewExpiring() targetsCacheTTL := 1 * time.Minute @@ -790,8 +809,7 @@ func Test_cachedTargetsManager_ListTargets(t *testing.T) { targetsCacheTTL: targetsCacheTTL, } - ctx := context.Background() - got, err := m.ListTargets(ctx, tt.args.tgARN) + got, err := m.ListTargets(ctx, makeTargetGroupBinding(tt.args.tgARN)) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) } else { @@ -1180,16 +1198,17 @@ func Test_cachedTargetsManager_refreshUnhealthyTargets(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() + ctx := context.Background() elbv2Client := services.NewMockELBV2(ctrl) for _, call := range tt.fields.describeTargetHealthWithContextCalls { elbv2Client.EXPECT().DescribeTargetHealthWithContext(gomock.Any(), call.req).Return(call.resp, call.err) + elbv2Client.EXPECT().AssumeRole(ctx, gomock.Any(), gomock.Any()).Return(elbv2Client) } m := &cachedTargetsManager{ elbv2Client: elbv2Client, } - ctx := context.Background() - got, err := m.refreshUnhealthyTargets(ctx, tt.args.tgARN, tt.args.cachedTargets) + got, err := m.refreshUnhealthyTargets(ctx, makeTargetGroupBinding(tt.args.tgARN), tt.args.cachedTargets) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) } else { @@ -1341,18 +1360,20 @@ func Test_cachedTargetsManager_listTargetsFromAWS(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) + ctx := context.Background() + defer ctrl.Finish() elbv2Client := services.NewMockELBV2(ctrl) for _, call := range tt.fields.describeTargetHealthWithContextCalls { elbv2Client.EXPECT().DescribeTargetHealthWithContext(gomock.Any(), call.req).Return(call.resp, call.err) + elbv2Client.EXPECT().AssumeRole(ctx, gomock.Any(), gomock.Any()).Return(elbv2Client) } m := &cachedTargetsManager{ elbv2Client: elbv2Client, } - ctx := context.Background() - got, err := m.listTargetsFromAWS(ctx, tt.args.tgARN, tt.args.targets) + got, err := m.listTargetsFromAWS(ctx, makeTargetGroupBinding(tt.args.tgARN), tt.args.targets) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) } else { diff --git a/pkg/targetgroupbinding/utils.go b/pkg/targetgroupbinding/utils.go index 65f3860964..4c1ba22f78 100644 --- a/pkg/targetgroupbinding/utils.go +++ b/pkg/targetgroupbinding/utils.go @@ -2,6 +2,7 @@ package targetgroupbinding import ( "fmt" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" @@ -16,8 +17,27 @@ const ( // Index Key for "ServiceReference" index. IndexKeyServiceRefName = "spec.serviceRef.name" + + // Annotation for IAM Role ARN to assume when calling AWS APIs. + AnnotationIamRoleArnToAssume = "alb.ingress.kubernetes.io/IamRoleArnToAssume" + + // Annotation for IAM Role External ID to use when calling AWS APIs. + AnnotationAssumeRoleExternalId = "alb.ingress.kubernetes.io/AssumeRoleExternalId" ) +// AnnotationsToFields converts annotations to fields. Currently it's tgb.Spec.IamRoleArnToAssume and tgb.Spec.AssumeRoleExternalId +func AnnotationsToFields(tgb *elbv2api.TargetGroupBinding) { + for key, value := range tgb.Annotations { + if key == AnnotationIamRoleArnToAssume { + tgb.Spec.IamRoleArnToAssume = value + } else { + if key == AnnotationAssumeRoleExternalId { + tgb.Spec.AssumeRoleExternalId = value + } + } + } +} + // BuildTargetHealthPodConditionType constructs the condition type for TargetHealth pod condition. func BuildTargetHealthPodConditionType(tgb *elbv2api.TargetGroupBinding) corev1.PodConditionType { return corev1.PodConditionType(fmt.Sprintf("%s/%s", TargetHealthPodConditionTypePrefix, tgb.Name)) diff --git a/test/framework/framework.go b/test/framework/framework.go index 4402f16d17..973dbcb3fe 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -8,6 +8,7 @@ import ( "k8s.io/client-go/rest" elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws" + "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/throttle" "sigs.k8s.io/aws-load-balancer-controller/test/framework/controller" "sigs.k8s.io/aws-load-balancer-controller/test/framework/helm" @@ -23,7 +24,7 @@ type Framework struct { Options Options RestCfg *rest.Config K8sClient client.Client - Cloud aws.Cloud + Cloud services.Cloud CTRLInstallationManager controller.InstallationManager NSManager k8sresources.NamespaceManager diff --git a/webhooks/elbv2/targetgroupbinding_mutator.go b/webhooks/elbv2/targetgroupbinding_mutator.go index 8a005927cb..8fd2472b78 100644 --- a/webhooks/elbv2/targetgroupbinding_mutator.go +++ b/webhooks/elbv2/targetgroupbinding_mutator.go @@ -2,15 +2,18 @@ package elbv2 import ( "context" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" awssdk "github.com/aws/aws-sdk-go-v2/aws" elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + "github.com/go-logr/logr" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" + "sigs.k8s.io/aws-load-balancer-controller/pkg/targetgroupbinding" "sigs.k8s.io/aws-load-balancer-controller/pkg/webhook" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -39,6 +42,7 @@ func (m *targetGroupBindingMutator) Prototype(_ admission.Request) (runtime.Obje func (m *targetGroupBindingMutator) MutateCreate(ctx context.Context, obj runtime.Object) (runtime.Object, error) { tgb := obj.(*elbv2api.TargetGroupBinding) + targetgroupbinding.AnnotationsToFields(tgb) if err := m.defaultingTargetType(ctx, tgb); err != nil { return nil, err } @@ -59,8 +63,7 @@ func (m *targetGroupBindingMutator) defaultingTargetType(ctx context.Context, tg if tgb.Spec.TargetType != nil { return nil } - tgARN := tgb.Spec.TargetGroupARN - sdkTargetType, err := m.obtainSDKTargetTypeFromAWS(ctx, tgARN) + sdkTargetType, err := m.obtainSDKTargetTypeFromAWS(ctx, tgb) if err != nil { return errors.Wrap(err, "couldn't determine TargetType") } @@ -82,7 +85,7 @@ func (m *targetGroupBindingMutator) defaultingIPAddressType(ctx context.Context, if tgb.Spec.IPAddressType != nil { return nil } - targetGroupIPAddressType, err := m.getTargetGroupIPAddressTypeFromAWS(ctx, tgb.Spec.TargetGroupARN) + targetGroupIPAddressType, err := m.getTargetGroupIPAddressTypeFromAWS(ctx, tgb) if err != nil { return errors.Wrap(err, "unable to get target group IP address type") } @@ -94,7 +97,7 @@ func (m *targetGroupBindingMutator) defaultingVpcID(ctx context.Context, tgb *el if tgb.Spec.VpcID != "" { return nil } - vpcId, err := m.getVpcIDFromAWS(ctx, tgb.Spec.TargetGroupARN) + vpcId, err := m.getVpcIDFromAWS(ctx, tgb) if err != nil { return errors.Wrap(err, "unable to get target group VpcID") } @@ -102,8 +105,8 @@ func (m *targetGroupBindingMutator) defaultingVpcID(ctx context.Context, tgb *el return nil } -func (m *targetGroupBindingMutator) obtainSDKTargetTypeFromAWS(ctx context.Context, tgARN string) (string, error) { - targetGroup, err := m.getTargetGroupFromAWS(ctx, tgARN) +func (m *targetGroupBindingMutator) obtainSDKTargetTypeFromAWS(ctx context.Context, tgb *elbv2api.TargetGroupBinding) (string, error) { + targetGroup, err := m.getTargetGroupFromAWS(ctx, tgb) if err != nil { return "", err } @@ -111,8 +114,8 @@ func (m *targetGroupBindingMutator) obtainSDKTargetTypeFromAWS(ctx context.Conte } // getTargetGroupIPAddressTypeFromAWS returns the target group IP address type of AWS target group -func (m *targetGroupBindingMutator) getTargetGroupIPAddressTypeFromAWS(ctx context.Context, tgARN string) (elbv2api.TargetGroupIPAddressType, error) { - targetGroup, err := m.getTargetGroupFromAWS(ctx, tgARN) +func (m *targetGroupBindingMutator) getTargetGroupIPAddressTypeFromAWS(ctx context.Context, tgb *elbv2api.TargetGroupBinding) (elbv2api.TargetGroupIPAddressType, error) { + targetGroup, err := m.getTargetGroupFromAWS(ctx, tgb) if err != nil { return "", err } @@ -128,11 +131,12 @@ func (m *targetGroupBindingMutator) getTargetGroupIPAddressTypeFromAWS(ctx conte return ipAddressType, nil } -func (m *targetGroupBindingMutator) getTargetGroupFromAWS(ctx context.Context, tgARN string) (*elbv2types.TargetGroup, error) { +func (m *targetGroupBindingMutator) getTargetGroupFromAWS(ctx context.Context, tgb *elbv2api.TargetGroupBinding) (*elbv2types.TargetGroup, error) { + tgARN := tgb.Spec.TargetGroupARN req := &elbv2sdk.DescribeTargetGroupsInput{ TargetGroupArns: []string{tgARN}, } - tgList, err := m.elbv2Client.DescribeTargetGroupsAsList(ctx, req) + tgList, err := m.elbv2Client.AssumeRole(ctx, tgb.Spec.IamRoleArnToAssume, tgb.Spec.AssumeRoleExternalId).DescribeTargetGroupsAsList(ctx, req) if err != nil { return nil, err } @@ -142,8 +146,8 @@ func (m *targetGroupBindingMutator) getTargetGroupFromAWS(ctx context.Context, t return &tgList[0], nil } -func (m *targetGroupBindingMutator) getVpcIDFromAWS(ctx context.Context, tgARN string) (string, error) { - targetGroup, err := m.getTargetGroupFromAWS(ctx, tgARN) +func (m *targetGroupBindingMutator) getVpcIDFromAWS(ctx context.Context, tgb *elbv2api.TargetGroupBinding) (string, error) { + targetGroup, err := m.getTargetGroupFromAWS(ctx, tgb) if err != nil { return "", err } diff --git a/webhooks/elbv2/targetgroupbinding_mutator_test.go b/webhooks/elbv2/targetgroupbinding_mutator_test.go index 1b6df32614..774195ce00 100644 --- a/webhooks/elbv2/targetgroupbinding_mutator_test.go +++ b/webhooks/elbv2/targetgroupbinding_mutator_test.go @@ -2,20 +2,33 @@ package elbv2 import ( "context" - elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "testing" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + awssdk "github.com/aws/aws-sdk-go-v2/aws" elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/go-logr/logr" "github.com/golang/mock/gomock" "github.com/pkg/errors" "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" "sigs.k8s.io/controller-runtime/pkg/log" ) +func makeTargetGroupBinding(tgARN string) *elbv2api.TargetGroupBinding { + return &elbv2api.TargetGroupBinding{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + Spec: elbv2api.TargetGroupBindingSpec{ + TargetGroupARN: tgARN, + }, + } +} + func Test_targetGroupBindingMutator_MutateCreate(t *testing.T) { type describeTargetGroupsAsListCall struct { req *elbv2sdk.DescribeTargetGroupsInput @@ -245,8 +258,10 @@ func Test_targetGroupBindingMutator_MutateCreate(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() elbv2Client := services.NewMockELBV2(ctrl) + ctx := context.Background() for _, call := range tt.fields.describeTargetGroupsAsListCalls { elbv2Client.EXPECT().DescribeTargetGroupsAsList(gomock.Any(), call.req).Return(call.resp, call.err).AnyTimes() + elbv2Client.EXPECT().AssumeRole(ctx, gomock.Any(), gomock.Any()).Return(elbv2Client).AnyTimes() } m := &targetGroupBindingMutator{ @@ -347,17 +362,20 @@ func Test_targetGroupBindingMutator_obtainSDKTargetTypeFromAWS(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) + ctx := context.Background() + defer ctrl.Finish() elbv2Client := services.NewMockELBV2(ctrl) for _, call := range tt.fields.describeTargetGroupsAsListCalls { elbv2Client.EXPECT().DescribeTargetGroupsAsList(gomock.Any(), call.req).Return(call.resp, call.err) + elbv2Client.EXPECT().AssumeRole(ctx, gomock.Any(), gomock.Any()).Return(elbv2Client).AnyTimes() } m := &targetGroupBindingMutator{ elbv2Client: elbv2Client, logger: logr.New(&log.NullLogSink{}), } - got, err := m.obtainSDKTargetTypeFromAWS(context.Background(), tt.args.tgARN) + got, err := m.obtainSDKTargetTypeFromAWS(context.Background(), makeTargetGroupBinding(tt.args.tgARN)) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) } else { @@ -474,17 +492,20 @@ func Test_targetGroupBindingMutator_getIPAddressTypeFromAWS(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) + ctx := context.Background() + defer ctrl.Finish() elbv2Client := services.NewMockELBV2(ctrl) for _, call := range tt.fields.describeTargetGroupsAsListCalls { elbv2Client.EXPECT().DescribeTargetGroupsAsList(gomock.Any(), call.req).Return(call.resp, call.err) + elbv2Client.EXPECT().AssumeRole(ctx, gomock.Any(), gomock.Any()).Return(elbv2Client).AnyTimes() } m := &targetGroupBindingMutator{ elbv2Client: elbv2Client, logger: logr.New(&log.NullLogSink{}), } - got, err := m.getTargetGroupIPAddressTypeFromAWS(context.Background(), tt.args.tgARN) + got, err := m.getTargetGroupIPAddressTypeFromAWS(context.Background(), makeTargetGroupBinding(tt.args.tgARN)) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) } else { @@ -557,17 +578,20 @@ func Test_targetGroupBindingMutator_obtainSDKVpcIDFromAWS(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) + ctx := context.Background() + defer ctrl.Finish() elbv2Client := services.NewMockELBV2(ctrl) for _, call := range tt.fields.describeTargetGroupsAsListCalls { elbv2Client.EXPECT().DescribeTargetGroupsAsList(gomock.Any(), call.req).Return(call.resp, call.err) + elbv2Client.EXPECT().AssumeRole(ctx, gomock.Any(), gomock.Any()).Return(elbv2Client).AnyTimes() } m := &targetGroupBindingMutator{ elbv2Client: elbv2Client, logger: logr.New(&log.NullLogSink{}), } - got, err := m.getVpcIDFromAWS(context.Background(), tt.args.tgARN) + got, err := m.getVpcIDFromAWS(context.Background(), makeTargetGroupBinding(tt.args.tgARN)) if tt.wantErr != nil { assert.EqualError(t, err, tt.wantErr.Error()) } else { diff --git a/webhooks/elbv2/targetgroupbinding_validator.go b/webhooks/elbv2/targetgroupbinding_validator.go index 367c52a381..92e74d9345 100644 --- a/webhooks/elbv2/targetgroupbinding_validator.go +++ b/webhooks/elbv2/targetgroupbinding_validator.go @@ -2,10 +2,11 @@ package elbv2 import ( "context" - elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "regexp" "strings" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + awssdk "github.com/aws/aws-sdk-go-v2/aws" elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/go-logr/logr" @@ -14,6 +15,7 @@ import ( elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/aws-load-balancer-controller/pkg/targetgroupbinding" "sigs.k8s.io/aws-load-balancer-controller/pkg/webhook" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -53,6 +55,7 @@ func (v *targetGroupBindingValidator) Prototype(_ admission.Request) (runtime.Ob func (v *targetGroupBindingValidator) ValidateCreate(ctx context.Context, obj runtime.Object) error { tgb := obj.(*elbv2api.TargetGroupBinding) + targetgroupbinding.AnnotationsToFields(tgb) if err := v.checkRequiredFields(tgb); err != nil { return err } @@ -74,6 +77,9 @@ func (v *targetGroupBindingValidator) ValidateCreate(ctx context.Context, obj ru func (v *targetGroupBindingValidator) ValidateUpdate(ctx context.Context, obj runtime.Object, oldObj runtime.Object) error { tgb := obj.(*elbv2api.TargetGroupBinding) oldTgb := oldObj.(*elbv2api.TargetGroupBinding) + + targetgroupbinding.AnnotationsToFields(tgb) + if err := v.checkRequiredFields(tgb); err != nil { return err } @@ -157,7 +163,7 @@ func (v *targetGroupBindingValidator) checkNodeSelector(tgb *elbv2api.TargetGrou // checkTargetGroupIPAddressType ensures IP address type matches with that on the AWS target group func (v *targetGroupBindingValidator) checkTargetGroupIPAddressType(ctx context.Context, tgb *elbv2api.TargetGroupBinding) error { - targetGroupIPAddressType, err := v.getTargetGroupIPAddressTypeFromAWS(ctx, tgb.Spec.TargetGroupARN) + targetGroupIPAddressType, err := v.getTargetGroupIPAddressTypeFromAWS(ctx, tgb) if err != nil { return errors.Wrap(err, "unable to get target group IP address type") } @@ -176,7 +182,7 @@ func (v *targetGroupBindingValidator) checkTargetGroupVpcID(ctx context.Context, if !vpcIDPatternRegex.MatchString(tgb.Spec.VpcID) { return errors.Errorf(vpcIDValidationErr, tgb.Spec.VpcID) } - vpcID, err := v.getVpcIDFromAWS(ctx, tgb.Spec.TargetGroupARN) + vpcID, err := v.getVpcIDFromAWS(ctx, tgb) if err != nil { return errors.Wrap(err, "unable to get target group VpcID") } @@ -187,8 +193,8 @@ func (v *targetGroupBindingValidator) checkTargetGroupVpcID(ctx context.Context, } // getTargetGroupIPAddressTypeFromAWS returns the target group IP address type of AWS target group -func (v *targetGroupBindingValidator) getTargetGroupIPAddressTypeFromAWS(ctx context.Context, tgARN string) (elbv2api.TargetGroupIPAddressType, error) { - targetGroup, err := v.getTargetGroupFromAWS(ctx, tgARN) +func (v *targetGroupBindingValidator) getTargetGroupIPAddressTypeFromAWS(ctx context.Context, tgb *elbv2api.TargetGroupBinding) (elbv2api.TargetGroupIPAddressType, error) { + targetGroup, err := v.getTargetGroupFromAWS(ctx, tgb) if err != nil { return "", err } @@ -205,11 +211,12 @@ func (v *targetGroupBindingValidator) getTargetGroupIPAddressTypeFromAWS(ctx con } // getTargetGroupFromAWS returns the AWS target group corresponding to the ARN -func (v *targetGroupBindingValidator) getTargetGroupFromAWS(ctx context.Context, tgARN string) (*elbv2types.TargetGroup, error) { +func (v *targetGroupBindingValidator) getTargetGroupFromAWS(ctx context.Context, tgb *elbv2api.TargetGroupBinding) (*elbv2types.TargetGroup, error) { + tgARN := tgb.Spec.TargetGroupARN req := &elbv2sdk.DescribeTargetGroupsInput{ TargetGroupArns: []string{tgARN}, } - tgList, err := v.elbv2Client.DescribeTargetGroupsAsList(ctx, req) + tgList, err := v.elbv2Client.AssumeRole(ctx, tgb.Spec.IamRoleArnToAssume, tgb.Spec.AssumeRoleExternalId).DescribeTargetGroupsAsList(ctx, req) if err != nil { return nil, err } @@ -219,8 +226,8 @@ func (v *targetGroupBindingValidator) getTargetGroupFromAWS(ctx context.Context, return &tgList[0], nil } -func (v *targetGroupBindingValidator) getVpcIDFromAWS(ctx context.Context, tgARN string) (string, error) { - targetGroup, err := v.getTargetGroupFromAWS(ctx, tgARN) +func (v *targetGroupBindingValidator) getVpcIDFromAWS(ctx context.Context, tgb *elbv2api.TargetGroupBinding) (string, error) { + targetGroup, err := v.getTargetGroupFromAWS(ctx, tgb) if err != nil { return "", err } diff --git a/webhooks/elbv2/targetgroupbinding_validator_test.go b/webhooks/elbv2/targetgroupbinding_validator_test.go index 7ce250047b..555477bbf8 100644 --- a/webhooks/elbv2/targetgroupbinding_validator_test.go +++ b/webhooks/elbv2/targetgroupbinding_validator_test.go @@ -4,12 +4,13 @@ import ( "context" "crypto/rand" "fmt" - elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" - "github.com/google/uuid" "math/big" "strings" "testing" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + "github.com/google/uuid" + awssdk "github.com/aws/aws-sdk-go-v2/aws" elbv2sdk "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/go-logr/logr" @@ -280,6 +281,8 @@ func Test_targetGroupBindingValidator_ValidateCreate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) + ctx := context.Background() + defer ctrl.Finish() k8sSchema := runtime.NewScheme() clientgoscheme.AddToScheme(k8sSchema) @@ -288,6 +291,7 @@ func Test_targetGroupBindingValidator_ValidateCreate(t *testing.T) { elbv2Client := services.NewMockELBV2(ctrl) for _, call := range tt.fields.describeTargetGroupsAsListCalls { elbv2Client.EXPECT().DescribeTargetGroupsAsList(gomock.Any(), call.req).Return(call.resp, call.err) + elbv2Client.EXPECT().AssumeRole(ctx, gomock.Any(), gomock.Any()).Return(elbv2Client).AnyTimes() } v := &targetGroupBindingValidator{ k8sClient: k8sClient, @@ -1319,6 +1323,8 @@ func Test_targetGroupBindingValidator_checkTargetGroupVpcID(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) + ctx := context.Background() + defer ctrl.Finish() k8sSchema := runtime.NewScheme() clientgoscheme.AddToScheme(k8sSchema) @@ -1327,6 +1333,7 @@ func Test_targetGroupBindingValidator_checkTargetGroupVpcID(t *testing.T) { elbv2Client := services.NewMockELBV2(ctrl) for _, call := range tt.fields.describeTargetGroupsAsListCalls { elbv2Client.EXPECT().DescribeTargetGroupsAsList(gomock.Any(), call.req).Return(call.resp, call.err) + elbv2Client.EXPECT().AssumeRole(ctx, gomock.Any(), gomock.Any()).Return(elbv2Client).AnyTimes() } v := &targetGroupBindingValidator{ k8sClient: k8sClient,