Keep nodes while deorbiter is running

pkg/apis/kubernikus/v1/kluster.go

@@ -64,10 +64,6 @@ func (k *Kluster) NeedsFinalizer(finalizer string) bool {
 }
 
 func (k *Kluster) HasFinalizer(finalizer string) bool {
-	if k.ObjectMeta.DeletionTimestamp == nil {
-		// not deleted. do not remove finalizers at this time
-		return false
-	}
 
 	for _, f := range k.ObjectMeta.Finalizers {
 		if f == finalizer {

pkg/controller/deorbit/controller.go

@@ -82,6 +82,9 @@ func (d *DeorbitReconciler) Reconcile(kluster *v1.Kluster) (bool, error) {
 		if kluster.TerminationProtection() {
 			return false, nil
 		}
+		if kluster.ObjectMeta.DeletionTimestamp == nil {
+			return false, nil //wait until the kluster is marked for deletion
+		}
 		if kluster.HasFinalizer(DeorbiterFinalizer) {
 			if err := d.deorbit(kluster); err != nil {
 				return false, err

pkg/controller/deorbit/deorbiter.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes"
 	core_v1 "k8s.io/api/core/v1"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/record"
@@ -88,6 +89,18 @@ func NewDeorbiter(kluster *v1.Kluster, stopCh <-chan struct{}, clients config.Cl
 func (d *ConcreteDeorbiter) DeletePersistentVolumeClaims() (deleted []core_v1.PersistentVolumeClaim, err error) {
 	deleted = []core_v1.PersistentVolumeClaim{}
 
+	// Cordon all nodes to avoid scheduling new pods that bind volumes
+	nodes, err := d.Client.CoreV1().Nodes().List(context.TODO(), meta_v1.ListOptions{})
+	if err != nil {
+		return deleted, fmt.Errorf("Failed to list nodes: %w", err)
+	}
+	for _, node := range nodes.Items {
+		_, err := d.Client.CoreV1().Nodes().Patch(context.TODO(), node.Name, types.JSONPatchType, []byte(`[{"op": "replace", "path": "/spec/unschedulable", "value": true}]`), meta_v1.PatchOptions{})
+		if err != nil {
+			return deleted, fmt.Errorf("Failed to cordon node %s: %w", node.Name, err)
+		}
+	}
+
 	pvcs, err := d.Client.CoreV1().PersistentVolumeClaims(meta_v1.NamespaceAll).List(context.TODO(), meta_v1.ListOptions{})
 	if err != nil {
 		return deleted, err
@@ -106,12 +119,26 @@ func (d *ConcreteDeorbiter) DeletePersistentVolumeClaims() (deleted []core_v1.Pe
 		if pv.Spec.Cinder == nil && pv.Spec.CSI == nil {
 			continue
 		}
-		deleted = append(deleted, pvc)
 
 		err = d.Client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, meta_v1.DeleteOptions{})
 		if err != nil {
 			return deleted, err
 		}
+		deleted = append(deleted, pvc)
+		//delete pods holding a reference to the pvc
+		pods, err := d.Client.CoreV1().Pods(pvc.Namespace).List(context.TODO(), meta_v1.ListOptions{})
+		if err != nil {
+			return deleted, err
+		}
+		for _, pod := range pods.Items {
+			for _, volume := range pod.Spec.Volumes {
+				if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == pvc.Name {
+					if err := d.Client.CoreV1().Pods(pvc.Namespace).Delete(context.TODO(), pod.Name, meta_v1.DeleteOptions{}); err != nil {
+						return deleted, fmt.Errorf("Failed to delete pod %s/%s: %w", pod.Namespace, pod.Name, err)
+					}
+				}
+			}
+		}
 	}
 
 	return deleted, err

pkg/controller/launch/controller.go

@@ -11,6 +11,7 @@ import (
 	v1 "github.com/sapcc/kubernikus/pkg/apis/kubernikus/v1"
 	"github.com/sapcc/kubernikus/pkg/controller/base"
 	"github.com/sapcc/kubernikus/pkg/controller/config"
+	"github.com/sapcc/kubernikus/pkg/controller/deorbit"
 	"github.com/sapcc/kubernikus/pkg/controller/metrics"
 	"github.com/sapcc/kubernikus/pkg/controller/nodeobservatory"
 	informers_kubernikus "github.com/sapcc/kubernikus/pkg/generated/informers/externalversions/kubernikus/v1"
@@ -91,6 +92,9 @@ func (lr *LaunchReconciler) Reconcile(kluster *v1.Kluster) (requeue bool, err er
 		if kluster.TerminationProtection() {
 			return false, nil
 		}
+		if kluster.HasFinalizer(deorbit.DeorbiterFinalizer) {
+			return false, nil //Wait for dorbiter so that volumes are detached before deleting nodes
+		}
 
 		return lr.terminatePools(kluster)
 	}