Merge pull request #17211 from justinsb/tolerate_resource_in_use_on_g…

…ce_migs

delete cluster: avoid logspam on GCE "resource in use" error

pkg/resources/gce/errors.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"errors"
+
+	"google.golang.org/api/googleapi"
+	"k8s.io/klog/v2"
+)
+
+// AsGoogleAPIError returns a googleapi.Error in the error chain, or false if none is found.
+func AsGoogleAPIError(err error) (*googleapi.Error, bool) {
+	var googleAPIError *googleapi.Error
+	if errors.As(err, &googleAPIError) {
+		return googleAPIError, true
+	} else {
+		return nil, false
+	}
+}
+
+// IsDependencyViolation checks if the error is a dependency violation.
+func IsDependencyViolation(err error) bool {
+	apiErr, ok := AsGoogleAPIError(err)
+	if !ok {
+		return false
+	}
+
+	for _, e := range apiErr.Errors {
+		switch e.Reason {
+		case "resourceInUseByAnotherResource":
+			return true
+		default:
+			klog.Infof("unexpected gce error code: %+v", e)
+		}
+	}
+
+	return false
+}

pkg/resources/ops/delete.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/kops/pkg/resources"
 	awsresources "k8s.io/kops/pkg/resources/aws"
+	gceresources "k8s.io/kops/pkg/resources/gce"
 	"k8s.io/kops/upup/pkg/fi"
 )
 
@@ -128,7 +129,7 @@ func DeleteResources(cloud fi.Cloud, resourceMap map[string]*resources.Resource,
 					}
 					if err != nil {
 						mutex.Lock()
-						if awsresources.IsDependencyViolation(err) {
+						if awsresources.IsDependencyViolation(err) || gceresources.IsDependencyViolation(err) {
 							fmt.Printf("%s\tstill has dependencies, will retry\n", human)
 							klog.V(4).Infof("resource %q generated a dependency error: %v", human, err)
 						} else {

upup/pkg/fi/cloudup/gce/wrappers.go

@@ -38,7 +38,7 @@ func DeleteInstanceGroupManager(c GCECloud, t *compute.InstanceGroupManager) err
 			klog.Infof("InstanceGroupManager not found, assuming deleted: %q", t.SelfLink)
 			return nil
 		}
-		return fmt.Errorf("error deleting InstanceGroupManager %s: %v", t.SelfLink, err)
+		return fmt.Errorf("error deleting InstanceGroupManager %s: %w", t.SelfLink, err)
 	}
 
 	return c.WaitForOp(op)
@@ -58,7 +58,7 @@ func DeleteInstanceTemplate(c GCECloud, selfLink string) error {
 			klog.Infof("instancetemplate not found, assuming deleted: %q", selfLink)
 			return nil
 		}
-		return fmt.Errorf("error deleting InstanceTemplate %s: %v", selfLink, err)
+		return fmt.Errorf("error deleting InstanceTemplate %s: %w", selfLink, err)
 	}
 
 	return c.WaitForOp(op)
@@ -78,7 +78,7 @@ func DeleteInstance(c GCECloud, instanceSelfLink string) error {
 			klog.Infof("Instance not found, assuming deleted: %q", instanceSelfLink)
 			return nil
 		}
-		return fmt.Errorf("error deleting Instance %s: %v", instanceSelfLink, err)
+		return fmt.Errorf("error deleting Instance %s: %w", instanceSelfLink, err)
 	}
 
 	return c.WaitForOp(op)
@@ -100,7 +100,7 @@ func ListManagedInstances(c GCECloud, igm *compute.InstanceGroupManager) ([]*com
 
 	instances, err := c.Compute().InstanceGroupManagers().ListManagedInstances(ctx, project, zoneName, igm.Name)
 	if err != nil {
-		return nil, fmt.Errorf("error listing ManagedInstances in %s: %v", igm.Name, err)
+		return nil, fmt.Errorf("error listing ManagedInstances in %s: %w", igm.Name, err)
 	}
 
 	return instances, nil