Merge pull request kubernetes#34298 from derekwaynecarr/ns-controller-panic

Automatic merge from submit-queue

Fix potential panic in namespace controller when rapidly create/delet…

Fixes kubernetes#33676

The theory is this could occur in either of the following scenarios:

1. HA environment where a GET to a different API server than what the WATCH was read from
1. In a many controller scenario (i.e. where multiple finalizers participate), a namespace that is created and deleted with the same name could trip up the other namespace controller to see a namespace with the same name that was not actually in a delete state.  Added checks to verify uid matches across retry operations.

/cc @liggitt @kubernetes/rh-cluster-infra

Author: Kubernetes Submit Queue

pkg/controller/namespace/namespace_controller_utils.go

@@ -71,6 +71,7 @@ func (o operationNotSupportedCache) isSupported(key operationKey) bool {
 type updateNamespaceFunc func(kubeClient clientset.Interface, namespace *api.Namespace) (*api.Namespace, error)
 
 // retryOnConflictError retries the specified fn if there was a conflict error
+// it will return an error if the UID for an object changes across retry operations.
 // TODO RetryOnConflict should be a generic concept in client code
 func retryOnConflictError(kubeClient clientset.Interface, namespace *api.Namespace, fn updateNamespaceFunc) (result *api.Namespace, err error) {
 	latestNamespace := namespace
@@ -82,10 +83,14 @@ func retryOnConflictError(kubeClient clientset.Interface, namespace *api.Namespa
 		if !errors.IsConflict(err) {
 			return nil, err
 		}
+		prevNamespace := latestNamespace
 		latestNamespace, err = kubeClient.Core().Namespaces().Get(latestNamespace.Name)
 		if err != nil {
 			return nil, err
 		}
+		if prevNamespace.UID != latestNamespace.UID {
+			return nil, fmt.Errorf("namespace uid has changed across retries")
+		}
 	}
 }
 
@@ -385,9 +390,19 @@ func syncNamespace(
 		return err
 	}
 
+	// the latest view of the namespace asserts that namespace is no longer deleting..
+	if namespace.DeletionTimestamp.IsZero() {
+		return nil
+	}
+
 	// if the namespace is already finalized, delete it
 	if finalized(namespace) {
-		err = kubeClient.Core().Namespaces().Delete(namespace.Name, nil)
+		var opts *api.DeleteOptions
+		uid := namespace.UID
+		if len(uid) > 0 {
+			opts = &api.DeleteOptions{Preconditions: &api.Preconditions{UID: &uid}}
+		}
+		err = kubeClient.Core().Namespaces().Delete(namespace.Name, opts)
 		if err != nil && !errors.IsNotFound(err) {
 			return err
 		}