awscloud: try to terminate previous secure instance

In case the previous executor SI belonging to the worker did not get
shut down properly, attempt to do it again when starting a new one,
otherwise replacing the SG or LT will not work.

internal/cloud/awscloud/secure-instance.go

@@ -78,6 +78,11 @@ func (a *AWS) RunSecureInstance(iamProfile, keyName, cloudWatchGroup, hostname s
 		}
 	}()
 
+	previousSI, err := a.terminatePreviousSI(identity.InstanceID)
+	if err != nil {
+		logrus.Warningf("Unable to terminate previous secure instance with instance ID (%s): %v, this job is now likely to fail", previousSI, err)
+	}
+
 	sgID, err := a.createOrReplaceSG(identity.InstanceID, identity.PrivateIP, vpcID)
 	if sgID != "" {
 		secureInstance.SGID = sgID
@@ -194,6 +199,42 @@ func (a *AWS) TerminateSecureInstance(si *SecureInstance) error {
 	return nil
 }
 
+func (a *AWS) terminatePreviousSI(hostInstanceID string) (string, error) {
+	descrInstancesOutput, err := a.ec2.DescribeInstances(&ec2.DescribeInstancesInput{
+		Filters: []*ec2.Filter{
+			&ec2.Filter{
+				Name: aws.String("tag:parent"),
+				Values: []*string{aws.String(hostInstanceID)},
+			},
+		},
+	})
+	if err != nil {
+		return "", err
+	}
+	if len(descrInstancesOutput.Reservations) == 0 || len(descrInstancesOutput.Reservations[0].Instances) == 0 {
+		return "", nil
+	}
+
+	if *descrInstancesOutput.Reservations[0].Instances[0].State.Name == ec2.InstanceStateNameTerminated {
+		return "", nil
+	}
+
+	instanceID := descrInstancesOutput.Reservations[0].Instances[0].InstanceId
+	_, err = a.ec2.TerminateInstances(&ec2.TerminateInstancesInput{
+		InstanceIds: []*string{instanceID},
+	})
+	if err != nil {
+		return *instanceID, err
+	}
+	err = a.ec2.WaitUntilInstanceTerminated(&ec2.DescribeInstancesInput{
+		InstanceIds: []*string{instanceID},
+	})
+	if err != nil {
+		return *instanceID, err
+	}
+	return *instanceID, nil
+}
+
 func isInvalidGroupNotFoundErr(err error) bool {
 	if awsErr, ok := err.(awserr.Error); ok {
 		if awsErr.Code() == "InvalidGroup.NotFound" {