EVEREST-1689 | Use new PSMDB finalizers (#603)

Signed-off-by: Mayank Shah <[email protected]>
percona · Jan 27, 2025 · aeee65c · aeee65c
1 parent cc7a946
commit aeee65c
Show file tree

Hide file tree

Showing 38 changed files with 125 additions and 71 deletions.
diff --git a/api/v1alpha1/databasecluster_types.go b/api/v1alpha1/databasecluster_types.go
@@ -113,6 +113,7 @@ type Applier interface {
 	DataSource() error
 	Monitoring() error
 	Backup() error
+	Metadata() error
 }
 
 // Storage is the storage configuration.

diff --git a/internal/controller/databasecluster_controller.go b/internal/controller/databasecluster_controller.go
@@ -154,6 +154,9 @@ func (r *DatabaseClusterReconciler) reconcileDB(
 		applier := p.Apply(ctx)
 		applier.Paused(db.Spec.Paused)
 		applier.AllowUnsafeConfig(db.Spec.AllowUnsafeConfiguration)
+		if err := applier.Metadata(); err != nil {
+			return err
+		}
 		if err := applier.Engine(); err != nil {
 			return err
 		}

diff --git a/internal/controller/databaseclusterbackup_controller.go b/internal/controller/databaseclusterbackup_controller.go
@@ -74,7 +74,7 @@ const (
 	pxcGapsReasonString               = "BinlogGapDetected"
 
 	deletePXCBackupFinalizer   = "delete-s3-backup"
-	deletePSMDBBackupFinalizer = "delete-backup"
+	deletePSMDBBackupFinalizer = "percona.com/delete-backup"
 )
 
 // ErrBackupStorageUndefined is returned when a backup storage is not defined
@@ -664,6 +664,15 @@ func (r *DatabaseClusterBackupReconciler) reconcilePSMDB(
 			return false, err
 		}
 	}
+	// replace legacy finalizer.
+	if controllerutil.RemoveFinalizer(psmdbCR, "delete-backup") &&
+		psmdbCR.GetDeletionTimestamp().IsZero() {
+		controllerutil.AddFinalizer(psmdbCR, deletePSMDBBackupFinalizer)
+		if err := r.Update(ctx, psmdbCR); err != nil {
+			return false, err
+		}
+	}
+
 	backup.Status.State = everestv1alpha1.GetDBBackupState(psmdbCR)
 	backup.Status.CompletedAt = psmdbCR.Status.CompletedAt
 	backup.Status.CreatedAt = &psmdbCR.CreationTimestamp

diff --git a/internal/controller/providers/pg/applier.go b/internal/controller/providers/pg/applier.go
@@ -65,6 +65,18 @@ func (p *applier) Paused(paused bool) {
 func (p *applier) AllowUnsafeConfig(_ bool) {
 }
 
+func (p *applier) Metadata() error {
+	if p.PerconaPGCluster.GetDeletionTimestamp().IsZero() {
+		for _, f := range []string{
+			finalizerDeletePGPVC,
+			finalizerDeletePGSSL,
+		} {
+			controllerutil.AddFinalizer(p.PerconaPGCluster, f)
+		}
+	}
+	return nil
+}
+
 func (p *applier) Engine() error {
 	pg := p.PerconaPGCluster
 	database := p.DB

diff --git a/internal/controller/providers/pg/provider.go b/internal/controller/providers/pg/provider.go
@@ -24,7 +24,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	everestv1alpha1 "github.com/percona/everest-operator/api/v1alpha1"
 	"github.com/percona/everest-operator/internal/controller/common"
@@ -66,14 +65,6 @@ func New(
 	currentPGSpec := pg.Spec
 	pg.Spec = defaultSpec()
 
-	finalizers := []string{
-		finalizerDeletePGPVC,
-		finalizerDeletePGSSL,
-	}
-	for _, f := range finalizers {
-		controllerutil.AddFinalizer(pg, f)
-	}
-
 	p := &Provider{
 		PerconaPGCluster: pg,
 		ProviderOptions:  opts,

diff --git a/internal/controller/providers/psmdb/applier.go b/internal/controller/providers/psmdb/applier.go
@@ -29,6 +29,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	everestv1alpha1 "github.com/percona/everest-operator/api/v1alpha1"
 	"github.com/percona/everest-operator/internal/controller/common"
@@ -48,6 +49,26 @@ type applier struct {
 	ctx context.Context //nolint:containedctx
 }
 
+func (p *applier) Metadata() error {
+	if p.PerconaServerMongoDB.GetDeletionTimestamp().IsZero() {
+		for _, f := range []string{
+			finalizerDeletePSMDBPodsInOrder,
+			finalizerDeletePSMDBPVC,
+		} {
+			controllerutil.AddFinalizer(p.PerconaServerMongoDB, f)
+		}
+
+		// remove legacy finalizers.
+		for _, f := range []string{
+			"delete-psmdb-pods-in-order",
+			"delete-psmdb-pvc",
+		} {
+			controllerutil.RemoveFinalizer(p.PerconaServerMongoDB, f)
+		}
+	}
+	return nil
+}
+
 func (p *applier) Paused(paused bool) {
 	p.PerconaServerMongoDB.Spec.Pause = paused
 }

diff --git a/internal/controller/providers/psmdb/provider.go b/internal/controller/providers/psmdb/provider.go
@@ -28,7 +28,6 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	everestv1alpha1 "github.com/percona/everest-operator/api/v1alpha1"
 	"github.com/percona/everest-operator/internal/controller/common"
@@ -37,8 +36,8 @@ import (
 )
 
 const (
-	finalizerDeletePSMDBPodsInOrder = "delete-psmdb-pods-in-order"
-	finalizerDeletePSMDBPVC         = "delete-psmdb-pvc"
+	finalizerDeletePSMDBPodsInOrder = "percona.com/delete-psmdb-pods-in-order"
+	finalizerDeletePSMDBPVC         = "percona.com/delete-psmdb-pvc"
 )
 
 // Provider is a provider for Percona Server for MongoDB.
@@ -71,15 +70,6 @@ func New(
 		return nil, err
 	}
 
-	// Add necessary finalizers.
-	finalizers := []string{
-		finalizerDeletePSMDBPodsInOrder,
-		finalizerDeletePSMDBPVC,
-	}
-	for _, f := range finalizers {
-		controllerutil.AddFinalizer(psmdb, f)
-	}
-
 	dbEngine, err := common.GetDatabaseEngine(ctx, client, common.PSMDBDeploymentName, opts.DB.GetNamespace())
 	if err != nil {
 		return nil, err

diff --git a/internal/controller/providers/pxc/applier.go b/internal/controller/providers/pxc/applier.go
@@ -44,6 +44,19 @@ type applier struct {
 	ctx context.Context //nolint:containedctx
 }
 
+func (p *applier) Metadata() error {
+	if p.PerconaXtraDBCluster.GetDeletionTimestamp().IsZero() {
+		for _, f := range []string{
+			finalizerDeletePXCPodsInOrder,
+			finalizerDeletePXCPVC,
+			finalizerDeletePXCSSL,
+		} {
+			controllerutil.AddFinalizer(p.PerconaXtraDBCluster, f)
+		}
+	}
+	return nil
+}
+
 func (p *applier) Paused(paused bool) {
 	p.Provider.PerconaXtraDBCluster.Spec.Pause = paused
 }

diff --git a/internal/controller/providers/pxc/provider.go b/internal/controller/providers/pxc/provider.go
@@ -28,7 +28,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	everestv1alpha1 "github.com/percona/everest-operator/api/v1alpha1"
 	"github.com/percona/everest-operator/internal/controller/common"
@@ -71,16 +70,6 @@ func New(
 		return nil, err
 	}
 
-	// Add necessary finalizers.
-	finalizers := []string{
-		finalizerDeletePXCPodsInOrder,
-		finalizerDeletePXCPVC,
-		finalizerDeletePXCSSL,
-	}
-	for _, f := range finalizers {
-		controllerutil.AddFinalizer(pxc, f)
-	}
-
 	dbEngine, err := common.GetDatabaseEngine(ctx, client, common.PXCDeploymentName, opts.DB.GetNamespace())
 	if err != nil {
 		return nil, err

diff --git a/tests/e2e/core/pg/40-delete-cluster.yaml b/tests/e2e/core/pg/40-delete-cluster.yaml
@@ -2,8 +2,9 @@ apiVersion: kuttl.dev/v1
 kind: TestStep
 timeout: 10
 commands:
-  - command: kubectl -n $NAMESPACE delete db test-pg-cluster
+  - script: kubectl -n $NAMESPACE delete db test-pg-cluster --wait=false && sleep 5
   - command: kubectl patch postgresclusters test-pg-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
     ignoreFailure: true
   - command: kubectl patch pg test-pg-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
     ignoreFailure: true
+  - command: kubectl wait --for=delete db test-pg-cluster --timeout=60s -n $NAMESPACE
diff --git a/tests/e2e/core/pg/90-delete-clusters.yaml b/tests/e2e/core/pg/90-delete-clusters.yaml
@@ -1,9 +1,11 @@
 apiVersion: kuttl.dev/v1
 kind: TestStep
 commands:
-  - command: kubectl -n $NAMESPACE delete db test-single-node
+  - script: kubectl -n $NAMESPACE delete db test-single-node --wait=false && sleep 5
   - command: kubectl patch postgresclusters test-single-node -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
     ignoreFailure: true
-  - command: kubectl -n $NAMESPACE delete db test-pg-cluster
+  - script: kubectl -n $NAMESPACE delete db test-pg-cluster --wait=false && sleep 5
   - command: kubectl patch postgresclusters test-pg-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
     ignoreFailure: true
+  - command: kubectl wait --for=delete db test-single-node --timeout=60s -n $NAMESPACE
+  - command: kubectl wait --for=delete db test-pg-cluster --timeout=60s -n $NAMESPACE
diff --git a/tests/e2e/core/psmdb/10-assert.yaml b/tests/e2e/core/psmdb/10-assert.yaml
@@ -35,8 +35,8 @@ kind: PerconaServerMongoDB
 metadata:
   name: test-psmdb-cluster
   finalizers:
-    - delete-psmdb-pods-in-order
-    - delete-psmdb-pvc
+    - percona.com/delete-psmdb-pods-in-order
+    - percona.com/delete-psmdb-pvc
 spec:
   image: percona/percona-server-mongodb:7.0.12-7
   multiCluster:

diff --git a/tests/e2e/core/psmdb/40-delete-clusters.yaml b/tests/e2e/core/psmdb/40-delete-clusters.yaml
@@ -1,8 +1,9 @@
 apiVersion: kuttl.dev/v1
 kind: TestStep
 commands:
-  - command: kubectl -n $NAMESPACE delete db test-psmdb-cluster
+  - script: kubectl -n $NAMESPACE delete db test-psmdb-cluster --wait=false && sleep 5
   - command: kubectl -n $NAMESPACE delete secret everest-secrets-test-psmdb-cluster
     ignoreFailure: true
   - command: kubectl patch psmdb test-psmdb-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
     ignoreFailure: true
+  - command: kubectl wait --for=delete db test-psmdb-cluster --timeout=60s -n $NAMESPACE
diff --git a/tests/e2e/core/psmdb/97-delete-clusters.yaml b/tests/e2e/core/psmdb/97-delete-clusters.yaml
@@ -2,6 +2,7 @@ apiVersion: kuttl.dev/v1
 kind: TestStep
 timeout: 10
 commands:
-  - command: kubectl -n $NAMESPACE delete db single-node
+  - script: kubectl -n $NAMESPACE delete db single-node --wait=false && sleep 5
   - command: kubectl patch psmdb single-node -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
     ignoreFailure: true
+  - command: kubectl wait --for=delete db single-node --timeout=60s -n $NAMESPACE
diff --git a/tests/e2e/core/pxc/40-delete-cluster.yaml b/tests/e2e/core/pxc/40-delete-cluster.yaml
@@ -2,6 +2,7 @@ apiVersion: kuttl.dev/v1
 kind: TestStep
 timeout: 10
 commands:
-  - command: kubectl -n $NAMESPACE delete db test-pxc-cluster
+  - script: kubectl -n $NAMESPACE delete db test-pxc-cluster --wait=false && sleep 5
   - command: kubectl patch pxc test-pxc-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
     ignoreFailure: true
+  - command: kubectl wait --for=delete db test-pxc-cluster --timeout=60s -n $NAMESPACE
diff --git a/tests/e2e/core/pxc/97-delete-cluster.yaml b/tests/e2e/core/pxc/97-delete-cluster.yaml
@@ -1,6 +1,7 @@
 apiVersion: kuttl.dev/v1
 kind: TestStep
 commands:
-  - command: kubectl -n $NAMESPACE delete db test-pxc-cluster
+  - script: kubectl -n $NAMESPACE delete db test-pxc-cluster --wait=false && sleep 5
   - command: kubectl patch pxc test-pxc-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
     ignoreFailure: true
+  - command: kubectl wait --for=delete db test-pxc-cluster --timeout=60s -n $NAMESPACE
diff --git a/tests/e2e/upgrade/pg/90-delete-clusters.yaml b/tests/e2e/upgrade/pg/90-delete-clusters.yaml
@@ -1,5 +1,7 @@
 apiVersion: kuttl.dev/v1
 kind: TestStep
 commands:
-  - command: kubectl patch perconapgclusters test-pg-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
-  - command: kubectl -n $NAMESPACE delete db test-pg-cluster
+  - script: kubectl -n $NAMESPACE delete db test-pg-cluster --wait=false && sleep 5
+  - script: kubectl patch perconapgclusters test-pg-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge || true
+  - command: kubectl wait --for=delete db test-pg-cluster --timeout=5m -n $NAMESPACE
+
diff --git a/tests/e2e/upgrade/psmdb/90-delete-cluster.yaml b/tests/e2e/upgrade/psmdb/90-delete-cluster.yaml
@@ -1,6 +1,6 @@
 apiVersion: kuttl.dev/v1
 kind: TestStep
 commands:
+  - script: kubectl -n $NAMESPACE delete db test-psmdb-cluster --wait=false && sleep 5
   - command: kubectl patch psmdb test-psmdb-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
-  - command: kubectl -n $NAMESPACE delete db test-psmdb-cluster
-    ignoreFailure: true
+  - command: kubectl wait --for=delete db test-psmdb-cluster --timeout=60s -n $NAMESPACE
diff --git a/tests/e2e/upgrade/pxc/90-delete-cluster.yaml b/tests/e2e/upgrade/pxc/90-delete-cluster.yaml
@@ -1,6 +1,6 @@
 apiVersion: kuttl.dev/v1
 kind: TestStep
 commands:
+  - script: kubectl -n $NAMESPACE delete db test-pxc-cluster --wait=false && sleep 5
   - command: kubectl patch pxc test-pxc-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
-  - command: kubectl -n $NAMESPACE delete db test-pxc-cluster
-    ignoreFailure: true
+  - command: kubectl wait --for=delete db test-pxc-cluster --timeout=60s -n $NAMESPACE
diff --git a/tests/integration/core/pg/19-delete-cluster.yaml b/tests/integration/core/pg/19-delete-cluster.yaml
@@ -1,5 +1,6 @@
 apiVersion: kuttl.dev/v1
 kind: TestStep
 commands:
+  - script: kubectl -n $NAMESPACE delete db test-pg-cluster --wait=false && sleep 5
   - command: kubectl patch perconapgclusters test-pg-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
-  - command: kubectl -n $NAMESPACE delete db test-pg-cluster
+  - command: kubectl wait --for=delete db/test-pg-cluster -n $NAMESPACE
diff --git a/tests/integration/core/pg/29-delete-cluster.yaml b/tests/integration/core/pg/29-delete-cluster.yaml
@@ -1,5 +1,6 @@
 apiVersion: kuttl.dev/v1
 kind: TestStep
 commands:
+  - script: kubectl -n $NAMESPACE delete db test-pg-cluster --wait=false && sleep 5
   - command: kubectl patch perconapgclusters test-pg-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
-  - command: kubectl -n $NAMESPACE delete db test-pg-cluster
+  - command: kubectl wait --for=delete db/test-pg-cluster -n $NAMESPACE
diff --git a/tests/integration/core/psmdb/10-assert.yaml b/tests/integration/core/psmdb/10-assert.yaml
@@ -25,8 +25,8 @@ kind: PerconaServerMongoDB
 metadata:
   name: test-psmdb-cluster
   finalizers:
-    - delete-psmdb-pods-in-order
-    - delete-psmdb-pvc
+    - percona.com/delete-psmdb-pods-in-order
+    - percona.com/delete-psmdb-pvc
 spec:
   backup:
     configuration:

diff --git a/tests/integration/core/psmdb/19-delete-cluster.yaml b/tests/integration/core/psmdb/19-delete-cluster.yaml
@@ -1,5 +1,6 @@
 apiVersion: kuttl.dev/v1
 kind: TestStep
 commands:
+  - script: kubectl -n $NAMESPACE delete db test-psmdb-cluster --wait=false && sleep 5
   - command: kubectl patch psmdb test-psmdb-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
-  - command: kubectl -n $NAMESPACE delete db test-psmdb-cluster
+  - command: kubectl wait --for=delete db/test-psmdb-cluster -n $NAMESPACE
diff --git a/tests/integration/core/psmdb/20-assert.yaml b/tests/integration/core/psmdb/20-assert.yaml
@@ -38,8 +38,8 @@ kind: PerconaServerMongoDB
 metadata:
   name: test-psmdb-cluster
   finalizers:
-    - delete-psmdb-pods-in-order
-    - delete-psmdb-pvc
+    - percona.com/delete-psmdb-pods-in-order
+    - percona.com/delete-psmdb-pvc
 spec:
   allowUnsafeConfigurations: true
   backup:

diff --git a/tests/integration/core/psmdb/29-delete-cluster.yaml b/tests/integration/core/psmdb/29-delete-cluster.yaml
@@ -1,5 +1,6 @@
 apiVersion: kuttl.dev/v1
 kind: TestStep
 commands:
+  - script: kubectl -n $NAMESPACE delete db test-psmdb-cluster --wait=false && sleep 5
   - command: kubectl patch psmdb test-psmdb-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
-  - command: kubectl -n $NAMESPACE delete db test-psmdb-cluster
+  - command: kubectl wait --for=delete db/test-psmdb-cluster -n $NAMESPACE
diff --git a/tests/integration/core/pxc/19-delete-cluster.yaml b/tests/integration/core/pxc/19-delete-cluster.yaml
@@ -1,5 +1,6 @@
 apiVersion: kuttl.dev/v1
 kind: TestStep
 commands:
+  - script: kubectl -n $NAMESPACE delete db test-pxc-cluster --wait=false && sleep 5
   - command: kubectl patch pxc test-pxc-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
-  - command: kubectl -n $NAMESPACE delete db test-pxc-cluster
+  - command: kubectl wait --for=delete db/test-pxc-cluster -n $NAMESPACE
diff --git a/tests/integration/core/pxc/29-delete-cluster.yaml b/tests/integration/core/pxc/29-delete-cluster.yaml
@@ -1,5 +1,6 @@
 apiVersion: kuttl.dev/v1
 kind: TestStep
 commands:
+  - script: kubectl -n $NAMESPACE delete db test-pxc-cluster --wait=false && sleep 5
   - command: kubectl patch pxc test-pxc-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
-  - command: kubectl -n $NAMESPACE delete db test-pxc-cluster
+  - command: kubectl wait --for=delete db/test-pxc-cluster -n $NAMESPACE
diff --git a/tests/integration/features/affinity/95-cleanup.yaml b/tests/integration/features/affinity/95-cleanup.yaml
@@ -1,9 +1,11 @@
 apiVersion: kuttl.dev/v1
 kind: TestStep
 commands:
+  - script: kubectl -n $NAMESPACE delete db test-pxc-cluster --wait=false && sleep 5
   - command: kubectl patch pxc test-pxc-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
     ignoreFailure: true
-  - command: kubectl -n $NAMESPACE delete db test-pxc-cluster
+  - script: kubectl -n $NAMESPACE delete db test-psmdb-cluster --wait=false && sleep 5
   - command: kubectl patch psmdb test-psmdb-cluster -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge
     ignoreFailure: true
-  - command: kubectl -n $NAMESPACE delete db test-psmdb-cluster
+  - command: kubectl wait --for=delete db/test-psmdb-cluster -n $NAMESPACE
+  - command: kubectl wait --for=delete db/test-pxc-cluster -n $NAMESPACE
diff --git a/tests/integration/features/dbbackup_pg/22-assert.yaml b/tests/integration/features/dbbackup_pg/22-assert.yaml
@@ -1,6 +1,6 @@
 apiVersion: kuttl.dev/v1
 kind: TestAssert
-timeout: 20
+timeout: 60
 commands:
   # check that there is no dbr/pg-restore CRs left
   - script: >