Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TargetGroupBindings can now manipulate target groups from different aws accounts #3691

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

TargetGroupBindings can now manipulate target groups from different aws accounts #3691

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions apis/elbv2/v1beta1/targetgroupbinding_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,14 @@ type TargetGroupBindingSpec struct {
// VpcID is the VPC of the TargetGroup. If unspecified, it will be automatically inferred.
// +optional
VpcID string `json:"vpcID,omitempty"`

// IAM Role ARN to assume when calling AWS APIs. Useful if the target group is in a different AWS account
// +optional
IamRoleArnToAssume string `json:"-"` // `json:"iamRoleArnToAssume,omitempty"`

// IAM Role ARN to assume when calling AWS APIs. Needed to assume a role in another account and prevent the confused deputy problem. https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html
// +optional
AssumeRoleExternalId string `json:"-"` // `json:"assumeRoleExternalId,omitempty"`
}

// TargetGroupBindingStatus defines the observed state of TargetGroupBinding
Expand Down
4 changes: 2 additions & 2 deletions controllers/ingress/group_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1"
"sigs.k8s.io/aws-load-balancer-controller/controllers/ingress/eventhandlers"
"sigs.k8s.io/aws-load-balancer-controller/pkg/annotations"
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws"
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy"
elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2"
Expand Down Expand Up @@ -43,7 +43,7 @@ const (
)

// NewGroupReconciler constructs new GroupReconciler
func NewGroupReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder,
func NewGroupReconciler(cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder,
finalizerManager k8s.FinalizerManager, networkingSGManager networkingpkg.SecurityGroupManager,
networkingSGReconciler networkingpkg.SecurityGroupReconciler, subnetsResolver networkingpkg.SubnetsResolver,
elbv2TaggingManager elbv2deploy.TaggingManager, controllerConfig config.ControllerConfig, backendSGProvider networkingpkg.BackendSGProvider,
Expand Down
4 changes: 2 additions & 2 deletions controllers/service/service_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"k8s.io/client-go/tools/record"
"sigs.k8s.io/aws-load-balancer-controller/controllers/service/eventhandlers"
"sigs.k8s.io/aws-load-balancer-controller/pkg/annotations"
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws"
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy"
elbv2deploy "sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2"
Expand All @@ -34,7 +34,7 @@ const (
controllerName = "service"
)

func NewServiceReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder,
func NewServiceReconciler(cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder,
finalizerManager k8s.FinalizerManager, networkingSGManager networking.SecurityGroupManager,
networkingSGReconciler networking.SecurityGroupReconciler, subnetsResolver networking.SubnetsResolver,
vpcInfoProvider networking.VPCInfoProvider, elbv2TaggingManager elbv2deploy.TaggingManager, controllerConfig config.ControllerConfig,
Expand Down
63 changes: 59 additions & 4 deletions docs/deploy/installation.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ The LBC is supported by AWS. Some clusters may be using the legacy "in-tree" fun
!!!question "Existing AWS ALB Ingress Controller users"
The AWS ALB Ingress controller must be uninstalled before installing the AWS Load Balancer Controller.
Please follow our [migration guide](upgrade/migrate_v1_v2.md) to do a migration.

!!!warning "When using AWS Load Balancer Controller v2.5+"
The AWS LBC provides a mutating webhook for service resources to set the `spec.loadBalancerClass` field for service of type LoadBalancer on create.
This makes the AWS LBC the **default controller for service** of type LoadBalancer. You can disable this feature and revert to set Cloud Controller Manager (in-tree controller) as the default by setting the helm chart value **enableServiceMutatorWebhook to false** with `--set enableServiceMutatorWebhook=false` .
The AWS LBC provides a mutating webhook for service resources to set the `spec.loadBalancerClass` field for service of type LoadBalancer on create.
This makes the AWS LBC the **default controller for service** of type LoadBalancer. You can disable this feature and revert to set Cloud Controller Manager (in-tree controller) as the default by setting the helm chart value **enableServiceMutatorWebhook to false** with `--set enableServiceMutatorWebhook=false` .
You will no longer be able to provision new Classic Load Balancer (CLB) from your kubernetes service unless you disable this feature. Existing CLB will continue to work fine.

## Supported Kubernetes versions
Expand All @@ -30,7 +30,7 @@ The LBC is supported by AWS. Some clusters may be using the legacy "in-tree" fun
Isolated clusters are clusters without internet access, and instead reply on VPC endpoints for all required connects.
When installing the AWS LBC in isolated clusters, you need to disable shield, waf and wafv2 via controller flags `--enable-shield=false, --enable-waf=false, --enable-wafv2=false`
### Using the Amazon EC2 instance metadata server version 2 (IMDSv2)
We recommend blocking the access to instance metadata by requiring the instance to use [IMDSv2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html) only. For more information, please refer to the AWS guidance [here](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node). If you are using the IMDSv2, set the hop limit to 2 or higher in order to allow the LBC to perform the metadata introspection.
We recommend blocking the access to instance metadata by requiring the instance to use [IMDSv2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html) only. For more information, please refer to the AWS guidance [here](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node). If you are using the IMDSv2, set the hop limit to 2 or higher in order to allow the LBC to perform the metadata introspection.

You can set the IMDSv2 as follows:
```
Expand Down Expand Up @@ -127,6 +127,10 @@ If you're not setting up IAM roles for service accounts, apply the IAM policies
curl -o iam-policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.11.0/docs/install/iam_policy.json
```

## Special IAM cases

### You only want the LBC to add and remove IPs to already existing target groups:

The following IAM permissions subset is for those using `TargetGroupBinding` only and don't plan to use the LBC to manage security group rules:

```
Expand All @@ -152,6 +156,57 @@ The following IAM permissions subset is for those using `TargetGroupBinding` onl
}
```

### You only want the LBC to add and remove IPs to already existing target groups, also in other accounts, assuming roles

On the other hand, if you plan to use the LBC to manage also target groups in different accounts, you will need to add `"sts:AssumeRole"` to your list of permissions, in other words:

```
{
"Statement": [
{
"Action": [
"ec2:DescribeVpcs",
"ec2:DescribeSecurityGroups",
"ec2:DescribeInstances",
"elasticloadbalancing:DescribeTargetGroups",
"elasticloadbalancing:DescribeTargetHealth",
"elasticloadbalancing:ModifyTargetGroup",
"elasticloadbalancing:ModifyTargetGroupAttributes",
"elasticloadbalancing:RegisterTargets",
"elasticloadbalancing:DeregisterTargets",
"sts:AssumeRole"
],
"Effect": "Allow",
"Resource": "*"
}
],
"Version": "2012-10-17"
}
```

The assumed roles will need the exactly the same permissions, without `"sts:AssumeRole"`. The assumed role will need a to allow to be assumed by the main role, something like this:

```
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::999999999999999:user/test-alb-controller"
},
"Action": "sts:AssumeRole",
"Condition": {
"StringEquals": {
"sts:ExternalId": "very-secret-string"
}
}
}
]
}
```

## Network configuration

Review the [worker nodes security group](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) docs. Your node security group must permit incoming traffic on TCP port 9443 from the Kubernetes control plane. This is needed for webhook access.
Expand Down
21 changes: 20 additions & 1 deletion docs/guide/targetgroupbinding/spec.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,29 @@ Kubernetes meta/v1.ObjectMeta
</em>
</td>
<td>
Refer to the Kubernetes API documentation for the fields of the
<table>
<tr><td><code>annotations</code></td><td>

<table>
<tr><td><code>alb.ingress.kubernetes.io/IamRoleArnToAssume</code><br><i>string</i></td>
<td><i>(Optional)</i> In case the target group is in a differet AWS account, you put here the role that needs to be assumed in order to manipulate the target group.
</td></tr>
<tr><td><code>alb.ingress.kubernetes.io/AssumeRoleExternalId</code><br><i>string</i></td>
<td><i>(Optional)</i> The external ID for the assume role operation. Optional, but recommended. It helps you to prevent the <a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html" target="_blank">confused deputy problem</a>.
</td></tr>
</table>

<tr><td colspan=2>
Refer to the Kubernetes API documentation for the other fields of the
<code>metadata</code> field.
</td></tr>
</table></td></tr>



</td>
</tr>

<tr>
<td>
<code>spec</code></br>
Expand Down
23 changes: 23 additions & 0 deletions docs/guide/targetgroupbinding/targetgroupbinding.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,29 @@ spec:
...
```

### AssumeRole

Sometimes the AWS LoadBalancer controller needs to manipulate target groups from different AWS accounts.
The way to do that is assuming a role from such account. There are annotations that can help you with that:

* `alb.ingress.kubernetes.io/IamRoleArnToAssume`: the ARN that you need to assume
* `alb.ingress.kubernetes.io/AssumeRoleExternalId`: the external ID for the assume role operation. Optional, but recommended. It helps you to prevent the confused deputy problem ( https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html )


## Sample YAML

```yaml
apiVersion: elbv2.k8s.aws/v1beta1
kind: TargetGroupBinding
metadata:
name: my-tgb
annotations:
alb.ingress.kubernetes.io/IamRoleArnToAssume: "arn:aws:iam::999999999999:role/alb-controller-policy-to-assume"
alb.ingress.kubernetes.io/AssumeRoleExternalId: "some-magic-string"
spec:
...
```

## MultiCluster Target Group
TargetGroupBinding CRD supports sharing the same target group ARN among multiple clusters. Setting this flag will ensure the controller only operates on targets within the cluster.

Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ require (
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/service/iam v1.36.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect
Expand Down
131 changes: 94 additions & 37 deletions pkg/aws/cloud.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,22 @@ package aws
import (
"context"
"fmt"
"log"
"net"
"os"
"strings"

awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
"github.com/aws/aws-sdk-go-v2/aws/ratelimit"
"github.com/aws/aws-sdk-go-v2/aws/retry"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/aws/aws-sdk-go-v2/service/sts"

smithymiddleware "github.com/aws/smithy-go/middleware"
"net"
"os"
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/throttle"
"sigs.k8s.io/aws-load-balancer-controller/pkg/version"
"strings"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
Expand All @@ -29,37 +34,8 @@ import (

const userAgent = "elbv2.k8s.aws"

type Cloud interface {
// EC2 provides API to AWS EC2
EC2() services.EC2

// ELBV2 provides API to AWS ELBV2
ELBV2() services.ELBV2

// ACM provides API to AWS ACM
ACM() services.ACM

// WAFv2 provides API to AWS WAFv2
WAFv2() services.WAFv2

// WAFRegional provides API to AWS WAFRegional
WAFRegional() services.WAFRegional

// Shield provides API to AWS Shield
Shield() services.Shield

// RGT provides API to AWS RGT
RGT() services.RGT

// Region for the kubernetes cluster
Region() string

// VpcID for the LoadBalancer resources.
VpcID() string
}

// NewCloud constructs new Cloud implementation.
func NewCloud(cfg CloudConfig, metricsCollector *aws_metrics.Collector, logger logr.Logger, awsClientsProvider provider.AWSClientsProvider) (Cloud, error) {
func NewCloud(cfg CloudConfig, metricsCollector *aws_metrics.Collector, logger logr.Logger, awsClientsProvider provider.AWSClientsProvider) (services.Cloud, error) {
hasIPv4 := true
addrs, err := net.InterfaceAddrs()
if err == nil {
Expand Down Expand Up @@ -138,17 +114,26 @@ func NewCloud(cfg CloudConfig, metricsCollector *aws_metrics.Collector, logger l
if err != nil {
return nil, errors.Wrap(err, "failed to get VPC ID")
}

cfg.VpcID = vpcID
return &defaultCloud{

thisObj := &defaultCloud{
cfg: cfg,
ec2: ec2Service,
elbv2: services.NewELBV2(awsClientsProvider),
acm: services.NewACM(awsClientsProvider),
wafv2: services.NewWAFv2(awsClientsProvider),
wafRegional: services.NewWAFRegional(awsClientsProvider, cfg.Region),
shield: services.NewShield(awsClientsProvider),
rgt: services.NewRGT(awsClientsProvider),
}, nil

assumeRoleElbV2: make(map[string]services.ELBV2),
awsClientsProvider: awsClientsProvider,
logger: logger,
}

thisObj.elbv2 = services.NewELBV2(awsClientsProvider, thisObj)

return thisObj, nil
}

func getVpcID(cfg CloudConfig, ec2Service services.EC2, ec2Metadata services.EC2Metadata, logger logr.Logger) (string, error) {
Expand Down Expand Up @@ -222,7 +207,7 @@ func inferVPCIDFromTags(ec2Service services.EC2, VpcNameTagKey string, VpcNameTa
return *vpcs[0].VpcId, nil
}

var _ Cloud = &defaultCloud{}
var _ services.Cloud = &defaultCloud{}

type defaultCloud struct {
cfg CloudConfig
Expand All @@ -234,6 +219,78 @@ type defaultCloud struct {
wafRegional services.WAFRegional
shield services.Shield
rgt services.RGT

assumeRoleElbV2 map[string]services.ELBV2
awsClientsProvider provider.AWSClientsProvider
logger logr.Logger
}

// returns ELBV2 client for the given assumeRoleArn, or the default ELBV2 client if assumeRoleArn is empty
func (c *defaultCloud) GetAssumedRoleELBV2(ctx context.Context, assumeRoleArn string, externalId string) services.ELBV2 {

if assumeRoleArn == "" {
return c.elbv2
}

assumedRoleELBV2, exists := c.assumeRoleElbV2[assumeRoleArn]
if exists {
return assumedRoleELBV2
}
c.logger.Info("awsCloud", "method", "GetAssumedRoleELBV2", "AssumeRoleArn", assumeRoleArn, "externalId", externalId)

////////////////
existingAwsConfig, _ := c.awsClientsProvider.GetAWSConfig(ctx, "GetAWSConfigForIAMRoleImpersonation")

sourceAccount := sts.NewFromConfig(*existingAwsConfig)
response, err := sourceAccount.AssumeRole(ctx, &sts.AssumeRoleInput{
RoleArn: aws.String(assumeRoleArn),
RoleSessionName: aws.String("aws-load-balancer-controller"),
ExternalId: aws.String(externalId),
})
if err != nil {
log.Fatalf("Unable to assume target role, %v. Attempting to use default client", err)
return c.elbv2
}
assumedRoleCreds := response.Credentials
newCreds := credentials.NewStaticCredentialsProvider(*assumedRoleCreds.AccessKeyId, *assumedRoleCreds.SecretAccessKey, *assumedRoleCreds.SessionToken)
newAwsConfig, err := config.LoadDefaultConfig(ctx, config.WithRegion(c.cfg.Region), config.WithCredentialsProvider(newCreds))
if err != nil {
log.Fatalf("Unable to load static credentials for service client config, %v. Attempting to use default client", err)
return c.elbv2
}

existingAwsConfig.Credentials = newAwsConfig.Credentials // response.Credentials

// // var assumedRoleCreds *stsTypes.Credentials = response.Credentials

// // Create config with target service client, using assumed role
// cfg, err = config.LoadDefaultConfig(ctx, config.WithRegion(region), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(*assumedRoleCreds.AccessKeyId, *assumedRoleCreds.SecretAccessKey, *assumedRoleCreds.SessionToken)))
// if err != nil {
// log.Fatalf("unable to load static credentials for service client config, %v", err)
// }

// ////////////////
// appCreds := stscreds.NewAssumeRoleProvider(client, assumeRoleArn)
// value, err := appCreds.Retrieve(context.TODO())
// if err != nil {
// // handle error
// }
// /////////

// ///////////// OLD
// creds := stscreds.NewCredentials(c.session, assumeRoleArn, func(p *stscreds.AssumeRoleProvider) {
// p.ExternalID = &externalId
// })
// //////////////

// c.awsConfig.Credentials = creds
// // newObj := services.NewELBV2(c.session, c, c.awsCFG)
// newObj := services.NewELBV2(*c.awsConfig, c.endpointsResolver, c)

newObj := services.NewELBV2(c.awsClientsProvider, c)
c.assumeRoleElbV2[assumeRoleArn] = newObj

return newObj
}

func (c *defaultCloud) EC2() services.EC2 {
Expand Down
Loading