Add capability to reap EC2 nodes not managed by ASG

Signed-off-by: Terence Ho <[email protected]>
keikoproj · Oct 29, 2022 · 82832f9 · 82832f9
1 parent ef0e942
commit 82832f9
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 13 deletions.
diff --git a/pkg/reaper/nodereaper/helpers.go b/pkg/reaper/nodereaper/helpers.go
@@ -116,17 +116,41 @@ func (ctx *ReaperContext) uncordonNode(name string, dryRun bool, ignoreDrainFail
 	return nil
 }
 
-func (ctx *ReaperContext) terminateInstance(w autoscalingiface.AutoScalingAPI, id string, nodeName string) error {
-
-	terminateInput := &autoscaling.TerminateInstanceInAutoScalingGroupInput{
-		InstanceId:                     &id,
-		ShouldDecrementDesiredCapacity: aws.Bool(false),
-	}
-
-	_, err := w.TerminateInstanceInAutoScalingGroup(terminateInput)
-	if err != nil {
-		return err
-	}
+func (ctx *ReaperContext) terminateInstance(w ReaperAwsAuth, id string, nodeName string) error {
+    describeInput := &autoscaling.DescribeAutoScalingInstancesInput{
+        InstanceIds: []*string{
+            aws.String(id),
+        },
+    }
+
+    response, derr := w.ASG.DescribeAutoScalingInstances(describeInput)
+    if derr != nil {
+        return derr
+    }
+
+    if len(response.AutoScalingInstances) == 0 {
+        terminateInput := &ec2.TerminateInstancesInput{
+            DryRun:      aws.Bool(false),
+            InstanceIds: []*string{
+                aws.String(id),
+            },
+        }
+
+        _, err := w.EC2.TerminateInstances(terminateInput)
+        if err != nil {
+            return err
+        }
+    } else {
+        terminateInput := &autoscaling.TerminateInstanceInAutoScalingGroupInput{
+            InstanceId:                     &id,
+            ShouldDecrementDesiredCapacity: aws.Bool(false),
+        }
+
+        _, err := w.ASG.TerminateInstanceInAutoScalingGroup(terminateInput)
+        if err != nil {
+            return err
+        }
+    }
 
 	if err := ctx.annotateNode(nodeName, stateAnnotationKey, terminatedStateName); err != nil {
 		log.Warnf("failed to update state annotation on node '%v'", nodeName)

diff --git a/pkg/reaper/nodereaper/nodereaper.go b/pkg/reaper/nodereaper/nodereaper.go
@@ -563,7 +563,7 @@ func (ctx *ReaperContext) reapOldNodes(w ReaperAwsAuth) error {
 
 		if !ctx.DryRun {
 			log.Infof("reaping old node %v -> %v", instance.NodeName, instance.InstanceID)
-			err = ctx.terminateInstance(w.ASG, instance.InstanceID, instance.NodeName)
+			err = ctx.terminateInstance(w, instance.InstanceID, instance.NodeName)
 			if err != nil {
 				return err
 			}
@@ -623,7 +623,7 @@ func (ctx *ReaperContext) reapUnhealthyNodes(w ReaperAwsAuth) error {
 		if !ctx.DryRun {
 			log.Infof("reaping unhealthy node %v -> %v", instance.NodeName, instance)
 
-			err = ctx.terminateInstance(w.ASG, instance.InstanceID, instance.NodeName)
+			err = ctx.terminateInstance(w, instance.InstanceID, instance.NodeName)
 			if err != nil {
 				return err
 			}