fix: redis cluster update as scale out (OT-CONTAINER-KIT#872)

* fix: redis cluster update as scale out Signed-off-by: drivebyer <[email protected]> * add e2e test step && remove unnecessary test assert Signed-off-by: drivebyer <[email protected]> * add newline Signed-off-by: drivebyer <[email protected]> * add newline Signed-off-by: drivebyer <[email protected]> --------- Signed-off-by: drivebyer <[email protected]>
ksmartdata · Apr 16, 2024 · 80e7f2f · 80e7f2f
1 parent 7b8c375
commit 80e7f2f
Show file tree

Hide file tree

Showing 11 changed files with 61 additions and 292 deletions.
diff --git a/api/status/redis-cluster_status.go b/api/status/redis-cluster_status.go
@@ -3,17 +3,17 @@ package status
 type RedisClusterState string
 
 const (
-	ReadyClusterReason                string = "RedisCluster is ready"
 	InitializingClusterLeaderReason   string = "RedisCluster is initializing leaders"
 	InitializingClusterFollowerReason string = "RedisCluster is initializing followers"
 	BootstrapClusterReason            string = "RedisCluster is bootstrapping"
+	ReadyClusterReason                string = "RedisCluster is ready"
 )
 
 // Status Field of the Redis Cluster
 const (
-	// RedisClusterReady means the RedisCluster is ready for use, we use redis-cli --cluster check 127.0.0.1:6379 to check the cluster status
-	RedisClusterReady        RedisClusterState = "Ready"
 	RedisClusterInitializing RedisClusterState = "Initializing"
 	RedisClusterBootstrap    RedisClusterState = "Bootstrap"
+	// RedisClusterReady means the RedisCluster is ready for use, we use redis-cli --cluster check 127.0.0.1:6379 to check the cluster status
+	RedisClusterReady RedisClusterState = "Ready"
 	// RedisClusterFailed       RedisClusterState = "Failed"
 )
diff --git a/controllers/rediscluster_controller.go b/controllers/rediscluster_controller.go
@@ -108,8 +108,9 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	}
 
 	// Mark the cluster status as initializing if there are no leader or follower nodes
-	if instance.Status.ReadyLeaderReplicas == 0 && instance.Status.ReadyFollowerReplicas == 0 {
-		err = k8sutils.UpdateRedisClusterStatus(instance, status.RedisClusterInitializing, status.InitializingClusterLeaderReason, 0, 0, r.Dk8sClient)
+	if (instance.Status.ReadyLeaderReplicas == 0 && instance.Status.ReadyFollowerReplicas == 0) ||
+		instance.Status.ReadyLeaderReplicas != leaderReplicas {
+		err = k8sutils.UpdateRedisClusterStatus(instance, status.RedisClusterInitializing, status.InitializingClusterLeaderReason, instance.Status.ReadyLeaderReplicas, instance.Status.ReadyFollowerReplicas, r.Dk8sClient)
 		if err != nil {
 			return ctrl.Result{}, err
 		}
@@ -141,8 +142,9 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
 
 	if redisLeaderInfo.Status.ReadyReplicas == leaderReplicas {
 		// Mark the cluster status as initializing if there are no follower nodes
-		if instance.Status.ReadyLeaderReplicas == 0 && instance.Status.ReadyFollowerReplicas == 0 {
-			err = k8sutils.UpdateRedisClusterStatus(instance, status.RedisClusterInitializing, status.InitializingClusterFollowerReason, leaderReplicas, 0, r.Dk8sClient)
+		if (instance.Status.ReadyLeaderReplicas == 0 && instance.Status.ReadyFollowerReplicas == 0) ||
+			instance.Status.ReadyFollowerReplicas != followerReplicas {
+			err = k8sutils.UpdateRedisClusterStatus(instance, status.RedisClusterInitializing, status.InitializingClusterFollowerReason, leaderReplicas, instance.Status.ReadyFollowerReplicas, r.Dk8sClient)
 			if err != nil {
 				return ctrl.Result{}, err
 			}

diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/chainsaw-test.yaml
@@ -11,246 +11,29 @@ spec:
         - create:
             file: cluster.yaml
         - assert:
-            file: ready-cluster.yaml
-        - assert:
-            file: ready-sts.yaml
-    - name: Sleep for five minutes
-      try:
-        - sleep:
-            duration: 5m
-    - name: Ping Cluster Nodes
-      try:
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-1 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-2 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-0 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-1 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-2 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-    - name: Check Cluster Slots
-      try:
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered.'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-1 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-2 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-0 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-1 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-2 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-    - name: Scale Redis Cluster
+            file: cluster-status-00.yaml
+
+    - name: Scale Out Redis Cluster
       try:
         - apply:
-            file: cluster-scale-up.yaml
-        - assert:
-            file: ready-cluster-scale-up.yaml
+            file: cluster-scale-out.yaml
+
+    - name: Check Status 00
+      try:
         - assert:
-            file: ready-sts-scale-up.yaml
-    - name: Sleep for five minutes
+            file: cluster-scale-out-status-00.yaml
+
+    - name: Check Status 01
       try:
-        - sleep:
-            duration: 5m
-    - name: Ping Scaled Cluster Nodes
+        - assert:
+            file: cluster-scale-out-status-01.yaml
+
+    - name: Check Status 02
       try:
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-1 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-2 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-3 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-4 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-5 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-0 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-1 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-2 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-3 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-4 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-5 --
-              redis-cli -c -p 6379 ping
-            check:
-              ($stdout=='PONG'): true
-    - name: Check Scaled Cluster Slots
+        - assert:
+            file: cluster-scale-out-status-02.yaml
+
+    - name: Check Status 03
       try:
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered.'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-1 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-2 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-3 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered.'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-4 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-5 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-0 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-1 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-2 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-3 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-4 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
-        - script:
-            content: >
-              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-5 --
-              redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered'
-            check:
-              ($stdout=='[OK] All 16384 slots covered.'): true
+        - assert:
+            file: cluster-scale-out-status-03.yaml
diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-00.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-00.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: RedisCluster
+metadata:
+  name: redis-cluster-v1beta2
+status:
+  state: Initializing
+  reason: RedisCluster is initializing leaders
+  readyLeaderReplicas: 3
+  readyFollowerReplicas: 3
diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-01.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-01.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: RedisCluster
+metadata:
+  name: redis-cluster-v1beta2
+status:
+  state: Initializing
+  reason: RedisCluster is initializing followers
+  readyLeaderReplicas: 6
+  readyFollowerReplicas: 3
diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-02.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-02.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: RedisCluster
+metadata:
+  name: redis-cluster-v1beta2
+status:
+  state: Bootstrap
+  reason: RedisCluster is bootstrapping
+  readyLeaderReplicas: 6
+  readyFollowerReplicas: 6
diff --git a/...redis-cluster/ready-cluster-scale-up.yaml → ...-cluster/cluster-scale-out-status-03.yaml b/...redis-cluster/ready-cluster-scale-up.yaml → ...-cluster/cluster-scale-out-status-03.yaml
@@ -5,5 +5,6 @@ metadata:
   name: redis-cluster-v1beta2
 status:
   state: Ready
+  reason: RedisCluster is ready
   readyLeaderReplicas: 6
   readyFollowerReplicas: 6
diff --git a/...aling/redis-cluster/cluster-scale-up.yaml → ...ling/redis-cluster/cluster-scale-out.yaml b/...aling/redis-cluster/cluster-scale-up.yaml → ...ling/redis-cluster/cluster-scale-out.yaml
diff --git a/.../scaling/redis-cluster/ready-cluster.yaml → ...ling/redis-cluster/cluster-status-00.yaml b/.../scaling/redis-cluster/ready-cluster.yaml → ...ling/redis-cluster/cluster-status-00.yaml
@@ -5,5 +5,6 @@ metadata:
   name: redis-cluster-v1beta2
 status:
   state: Ready
+  reason: RedisCluster is ready
   readyLeaderReplicas: 3
   readyFollowerReplicas: 3
diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-sts-scale-up.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-sts-scale-up.yaml