update aws role management and cleanup role on dataplane delation (#105)

* update aws role management and cleanup Signed-off-by: Pulak Kanti Bhowmick <[email protected]> * detach policy before deleting role Signed-off-by: Pulak Kanti Bhowmick <[email protected]> --------- Signed-off-by: Pulak Kanti Bhowmick <[email protected]>
baazhq · Sep 14, 2024 · 4c30581 · 4c30581
1 parent cade15d
commit 4c30581
Show file tree

Hide file tree

Showing 9 changed files with 190 additions and 95 deletions.
diff --git a/api/v1/types/awscloudinfra_types.go b/api/v1/types/awscloudinfra_types.go
@@ -24,16 +24,17 @@ type EksConfig struct {
 }
 
 type AwsCloudInfraConfigStatus struct {
-	Vpc                string    `json:"vpc,omitempty"`
-	SubnetIds          []string  `json:"subnetIds,omitempty"`
-	SecurityGroupIds   []string  `json:"securityGroupIds,omitempty"`
-	NATGatewayId       string    `json:"natGatewayId,omitempty"`
-	NATAttachedWithRT  bool      `json:"natAttchedWithRT,omitempty"`
-	SGInboundRuleAdded bool      `json:"sgInboundRuleAdded,omitempty"`
-	InternetGatewayId  string    `json:"internetGatewayId,omitempty"`
-	PublicRTId         string    `json:"publicRTId,omitempty"`
-	LBArns             []string  `json:"lbArns,omitempty"`
-	EksStatus          EksStatus `json:"eksStatus,omitempty"`
+	Vpc                string          `json:"vpc,omitempty"`
+	SubnetIds          []string        `json:"subnetIds,omitempty"`
+	SecurityGroupIds   []string        `json:"securityGroupIds,omitempty"`
+	NATGatewayId       string          `json:"natGatewayId,omitempty"`
+	NATAttachedWithRT  bool            `json:"natAttchedWithRT,omitempty"`
+	SGInboundRuleAdded bool            `json:"sgInboundRuleAdded,omitempty"`
+	InternetGatewayId  string          `json:"internetGatewayId,omitempty"`
+	PublicRTId         string          `json:"publicRTId,omitempty"`
+	LBArns             []string        `json:"lbArns,omitempty"`
+	EksStatus          EksStatus       `json:"eksStatus,omitempty"`
+	Roles              map[string]bool `json:"roles,omitempty"`
 }
 
 type EksStatus struct {

diff --git a/api/v1/types/zz_generated.deepcopy.go b/api/v1/types/zz_generated.deepcopy.go
diff --git a/chart/baaz/crds/baaz.dev_dataplanes.yaml b/chart/baaz/crds/baaz.dev_dataplanes.yaml
@@ -164,6 +164,10 @@ spec:
                     type: string
                   publicRTId:
                     type: string
+                  roles:
+                    additionalProperties:
+                      type: boolean
+                    type: object
                   securityGroupIds:
                     items:
                       type: string

diff --git a/config/crd/bases/baaz.dev_dataplanes.yaml b/config/crd/bases/baaz.dev_dataplanes.yaml
@@ -164,6 +164,10 @@ spec:
                     type: string
                   publicRTId:
                     type: string
+                  roles:
+                    additionalProperties:
+                      type: boolean
+                    type: object
                   securityGroupIds:
                     items:
                       type: string

diff --git a/internal/dataplane_controller/aws_eks.go b/internal/dataplane_controller/aws_eks.go
@@ -255,6 +255,17 @@ func (ae *awsEnv) reconcileAwsEks() error {
 				return fmt.Errorf("failed to create cluster iam role: %s", err.Error())
 			}
 
+			if _, _, err = utils.PatchStatus(ae.ctx, ae.client, ae.dp, func(obj client.Object) client.Object {
+				ob := obj.(*v1.DataPlanes)
+				if ob.Status.CloudInfraStatus.Roles == nil {
+					ob.Status.CloudInfraStatus.Roles = make(map[string]bool)
+				}
+				ob.Status.CloudInfraStatus.Roles[*clusterRoleOutput.Role.RoleName] = true
+				return ob
+			}); err != nil {
+				return err
+			}
+
 			klog.Infof("Cluster Role [%s] Created", *clusterRoleOutput.Role.RoleName)
 
 			createEksResult := ae.eksIC.CreateEks()
@@ -1084,6 +1095,17 @@ func (ae *awsEnv) reconcileSystemNodeGroup() error {
 		return errors.New("node role is nil")
 	}
 
+	if _, _, err = utils.PatchStatus(ae.ctx, ae.client, ae.dp, func(obj client.Object) client.Object {
+		ob := obj.(*v1.DataPlanes)
+		if ob.Status.CloudInfraStatus.Roles == nil {
+			ob.Status.CloudInfraStatus.Roles = make(map[string]bool)
+		}
+		ob.Status.CloudInfraStatus.Roles[*nodeRole.Role.RoleName] = true
+		return ob
+	}); err != nil {
+		return err
+	}
+
 	subnetIds := ae.dp.Spec.CloudInfra.AwsCloudInfraConfig.Eks.SubnetIds
 	if ae.dp.Spec.CloudInfra.ProvisionNetwork {
 		subnetIds = ae.dp.Status.CloudInfraStatus.SubnetIds
@@ -1189,6 +1211,17 @@ func (ae *awsEnv) ReconcileDefaultAddons() error {
 				return err
 			}
 
+			if _, _, err = utils.PatchStatus(ae.ctx, ae.client, ae.dp, func(obj client.Object) client.Object {
+				ob := obj.(*v1.DataPlanes)
+				if ob.Status.CloudInfraStatus.Roles == nil {
+					ob.Status.CloudInfraStatus.Roles = make(map[string]bool)
+				}
+				ob.Status.CloudInfraStatus.Roles[*role.Role.RoleName] = true
+				return ob
+			}); err != nil {
+				return err
+			}
+
 			_, cErr := ae.eksIC.CreateAddon(ae.ctx, &awseks.CreateAddonInput{
 				AddonName:             aws.String(awsEbsCsiDriver),
 				ClusterName:           aws.String(clusterName),
@@ -1217,19 +1250,30 @@ func (ae *awsEnv) ReconcileDefaultAddons() error {
 		var notFoundErr *types.ResourceNotFoundException
 		if errors.As(err, &notFoundErr) {
 			klog.Info("Creating vpc cni addon")
-			_, arn, err := ae.eksIC.CreateVpcCniRole(ae.ctx)
+			role, arn, err := ae.eksIC.CreateVpcCniRole(ae.ctx)
 			if err != nil {
 				return err
 			}
 
+			if _, _, err = utils.PatchStatus(ae.ctx, ae.client, ae.dp, func(obj client.Object) client.Object {
+				ob := obj.(*v1.DataPlanes)
+				if ob.Status.CloudInfraStatus.Roles == nil {
+					ob.Status.CloudInfraStatus.Roles = make(map[string]bool)
+				}
+				ob.Status.CloudInfraStatus.Roles[*role.Role.RoleName] = true
+				return ob
+			}); err != nil {
+				return err
+			}
+
 			v := `{"enableNetworkPolicy": "true"}`
 
 			_, cErr := ae.eksIC.CreateAddon(ae.ctx, &awseks.CreateAddonInput{
 				AddonName:             aws.String(vpcCni),
 				ClusterName:           aws.String(clusterName),
 				ResolveConflicts:      types.ResolveConflictsOverwrite,
 				ServiceAccountRoleArn: aws.String(arn),
-				AddonVersion:          aws.String("v1.15.0-eksbuild.2"),
+				AddonVersion:          aws.String("v1.18.3-eksbuild.3"),
 				ConfigurationValues:   aws.String(v),
 			})
 			if cErr != nil {

diff --git a/internal/dataplane_controller/dataplane_controller.go b/internal/dataplane_controller/dataplane_controller.go
@@ -321,6 +321,13 @@ func (r *DataPlaneReconciler) reconcileDelete(ae *awsEnv) (ctrl.Result, error) {
 		return ctrl.Result{}, retryErr
 	}
 
+	// delete created roles
+	for roleName, _ := range ae.dp.Status.CloudInfraStatus.Roles {
+		if err := ae.eksIC.DeleteRole(ae.ctx, roleName); err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+
 	// update namespace level
 	customerNs := &core.Namespace{}
 	if err := ae.client.Get(ae.ctx, client.ObjectKey{Name: ae.dp.Namespace}, customerNs); err != nil {

diff --git a/internal/tenantinfra_controller/aws_eks.go b/internal/tenantinfra_controller/aws_eks.go
@@ -11,10 +11,6 @@ import (
 	awseks "github.com/aws/aws-sdk-go-v2/service/eks"
 	"github.com/aws/aws-sdk-go-v2/service/eks/types"
 	"github.com/aws/aws-sdk-go/aws"
-	v1 "github.com/baazhq/baaz/api/v1/types"
-	"github.com/baazhq/baaz/pkg/aws/eks"
-	"github.com/baazhq/baaz/pkg/store"
-	"github.com/baazhq/baaz/pkg/utils"
 	core "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -23,6 +19,11 @@ import (
 	"k8s.io/utils/strings/slices"
 	"sigs.k8s.io/aws-iam-authenticator/pkg/token"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	v1 "github.com/baazhq/baaz/api/v1/types"
+	"github.com/baazhq/baaz/pkg/aws/eks"
+	"github.com/baazhq/baaz/pkg/store"
+	"github.com/baazhq/baaz/pkg/utils"
 )
 
 type nodeGroupType string

diff --git a/pkg/aws/eks/interface.go b/pkg/aws/eks/interface.go
@@ -7,9 +7,10 @@ import (
 	awseks "github.com/aws/aws-sdk-go-v2/service/eks"
 	awsiam "github.com/aws/aws-sdk-go-v2/service/iam"
 	awssts "github.com/aws/aws-sdk-go-v2/service/sts"
-	v1 "github.com/baazhq/baaz/api/v1/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+
+	v1 "github.com/baazhq/baaz/api/v1/types"
 )
 
 type Eks interface {
@@ -44,6 +45,7 @@ type Eks interface {
 	DescribeInstances(ctx context.Context, input *awsec2.DescribeInstancesInput) (*awsec2.DescribeInstancesOutput, error)
 	CreateIAMPolicy(ctx context.Context, input *awsiam.CreatePolicyInput) (*awsiam.CreatePolicyOutput, error)
 	AttachRolePolicy(ctx context.Context, input *awsiam.AttachRolePolicyInput) (*awsiam.AttachRolePolicyOutput, error)
+	DeleteRole(ctx context.Context, roleName string) error
 }
 
 type eks struct {