From 80e7f2fd3b647bfacda8a42f1702c2d21e1e4346 Mon Sep 17 00:00:00 2001 From: yangw Date: Tue, 16 Apr 2024 21:44:03 +0800 Subject: [PATCH] fix: redis cluster update as scale out (#872) * fix: redis cluster update as scale out Signed-off-by: drivebyer * add e2e test step && remove unnecessary test assert Signed-off-by: drivebyer * add newline Signed-off-by: drivebyer * add newline Signed-off-by: drivebyer --------- Signed-off-by: drivebyer --- api/status/redis-cluster_status.go | 6 +- controllers/rediscluster_controller.go | 10 +- .../scaling/redis-cluster/chainsaw-test.yaml | 257 ++---------------- .../cluster-scale-out-status-00.yaml | 10 + .../cluster-scale-out-status-01.yaml | 10 + .../cluster-scale-out-status-02.yaml | 10 + ....yaml => cluster-scale-out-status-03.yaml} | 1 + ...r-scale-up.yaml => cluster-scale-out.yaml} | 0 ...dy-cluster.yaml => cluster-status-00.yaml} | 1 + .../redis-cluster/ready-sts-scale-up.yaml | 24 -- .../scaling/redis-cluster/ready-sts.yaml | 24 -- 11 files changed, 61 insertions(+), 292 deletions(-) create mode 100644 tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-00.yaml create mode 100644 tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-01.yaml create mode 100644 tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-02.yaml rename tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/{ready-cluster-scale-up.yaml => cluster-scale-out-status-03.yaml} (85%) rename tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/{cluster-scale-up.yaml => cluster-scale-out.yaml} (100%) rename tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/{ready-cluster.yaml => cluster-status-00.yaml} (85%) delete mode 100644 tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-sts-scale-up.yaml delete mode 100644 tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-sts.yaml diff --git a/api/status/redis-cluster_status.go b/api/status/redis-cluster_status.go index 6ccf24da2..e58c62ea5 100644 --- a/api/status/redis-cluster_status.go +++ b/api/status/redis-cluster_status.go @@ -3,17 +3,17 @@ package status type RedisClusterState string const ( - ReadyClusterReason string = "RedisCluster is ready" InitializingClusterLeaderReason string = "RedisCluster is initializing leaders" InitializingClusterFollowerReason string = "RedisCluster is initializing followers" BootstrapClusterReason string = "RedisCluster is bootstrapping" + ReadyClusterReason string = "RedisCluster is ready" ) // Status Field of the Redis Cluster const ( - // RedisClusterReady means the RedisCluster is ready for use, we use redis-cli --cluster check 127.0.0.1:6379 to check the cluster status - RedisClusterReady RedisClusterState = "Ready" RedisClusterInitializing RedisClusterState = "Initializing" RedisClusterBootstrap RedisClusterState = "Bootstrap" + // RedisClusterReady means the RedisCluster is ready for use, we use redis-cli --cluster check 127.0.0.1:6379 to check the cluster status + RedisClusterReady RedisClusterState = "Ready" // RedisClusterFailed RedisClusterState = "Failed" ) diff --git a/controllers/rediscluster_controller.go b/controllers/rediscluster_controller.go index c15fbb96b..2b07657b2 100644 --- a/controllers/rediscluster_controller.go +++ b/controllers/rediscluster_controller.go @@ -108,8 +108,9 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request } // Mark the cluster status as initializing if there are no leader or follower nodes - if instance.Status.ReadyLeaderReplicas == 0 && instance.Status.ReadyFollowerReplicas == 0 { - err = k8sutils.UpdateRedisClusterStatus(instance, status.RedisClusterInitializing, status.InitializingClusterLeaderReason, 0, 0, r.Dk8sClient) + if (instance.Status.ReadyLeaderReplicas == 0 && instance.Status.ReadyFollowerReplicas == 0) || + instance.Status.ReadyLeaderReplicas != leaderReplicas { + err = k8sutils.UpdateRedisClusterStatus(instance, status.RedisClusterInitializing, status.InitializingClusterLeaderReason, instance.Status.ReadyLeaderReplicas, instance.Status.ReadyFollowerReplicas, r.Dk8sClient) if err != nil { return ctrl.Result{}, err } @@ -141,8 +142,9 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request if redisLeaderInfo.Status.ReadyReplicas == leaderReplicas { // Mark the cluster status as initializing if there are no follower nodes - if instance.Status.ReadyLeaderReplicas == 0 && instance.Status.ReadyFollowerReplicas == 0 { - err = k8sutils.UpdateRedisClusterStatus(instance, status.RedisClusterInitializing, status.InitializingClusterFollowerReason, leaderReplicas, 0, r.Dk8sClient) + if (instance.Status.ReadyLeaderReplicas == 0 && instance.Status.ReadyFollowerReplicas == 0) || + instance.Status.ReadyFollowerReplicas != followerReplicas { + err = k8sutils.UpdateRedisClusterStatus(instance, status.RedisClusterInitializing, status.InitializingClusterFollowerReason, leaderReplicas, instance.Status.ReadyFollowerReplicas, r.Dk8sClient) if err != nil { return ctrl.Result{}, err } diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/chainsaw-test.yaml index 6eafeb05d..73085d4f9 100644 --- a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/chainsaw-test.yaml @@ -11,246 +11,29 @@ spec: - create: file: cluster.yaml - assert: - file: ready-cluster.yaml - - assert: - file: ready-sts.yaml - - name: Sleep for five minutes - try: - - sleep: - duration: 5m - - name: Ping Cluster Nodes - try: - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-1 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-2 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-0 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-1 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-2 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - name: Check Cluster Slots - try: - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered.' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-1 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-2 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-0 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-1 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-2 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - name: Scale Redis Cluster + file: cluster-status-00.yaml + + - name: Scale Out Redis Cluster try: - apply: - file: cluster-scale-up.yaml - - assert: - file: ready-cluster-scale-up.yaml + file: cluster-scale-out.yaml + + - name: Check Status 00 + try: - assert: - file: ready-sts-scale-up.yaml - - name: Sleep for five minutes + file: cluster-scale-out-status-00.yaml + + - name: Check Status 01 try: - - sleep: - duration: 5m - - name: Ping Scaled Cluster Nodes + - assert: + file: cluster-scale-out-status-01.yaml + + - name: Check Status 02 try: - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-1 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-2 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-3 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-4 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-5 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-0 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-1 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-2 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-3 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-4 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-5 -- - redis-cli -c -p 6379 ping - check: - ($stdout=='PONG'): true - - name: Check Scaled Cluster Slots + - assert: + file: cluster-scale-out-status-02.yaml + + - name: Check Status 03 try: - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered.' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-1 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-2 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-3 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered.' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-4 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-5 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-0 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-1 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-2 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-3 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-4 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true - - script: - content: > - kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-5 -- - redis-cli --cluster check 127.0.0.1:6379 | grep 'All 16384 slots covered' - check: - ($stdout=='[OK] All 16384 slots covered.'): true + - assert: + file: cluster-scale-out-status-03.yaml diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-00.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-00.yaml new file mode 100644 index 000000000..911b0b396 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-00.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisCluster +metadata: + name: redis-cluster-v1beta2 +status: + state: Initializing + reason: RedisCluster is initializing leaders + readyLeaderReplicas: 3 + readyFollowerReplicas: 3 diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-01.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-01.yaml new file mode 100644 index 000000000..de83a1cfe --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-01.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisCluster +metadata: + name: redis-cluster-v1beta2 +status: + state: Initializing + reason: RedisCluster is initializing followers + readyLeaderReplicas: 6 + readyFollowerReplicas: 3 diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-02.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-02.yaml new file mode 100644 index 000000000..486807c1d --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-02.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisCluster +metadata: + name: redis-cluster-v1beta2 +status: + state: Bootstrap + reason: RedisCluster is bootstrapping + readyLeaderReplicas: 6 + readyFollowerReplicas: 6 diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-cluster-scale-up.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-03.yaml similarity index 85% rename from tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-cluster-scale-up.yaml rename to tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-03.yaml index eb06df309..593fde3e3 100644 --- a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-cluster-scale-up.yaml +++ b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out-status-03.yaml @@ -5,5 +5,6 @@ metadata: name: redis-cluster-v1beta2 status: state: Ready + reason: RedisCluster is ready readyLeaderReplicas: 6 readyFollowerReplicas: 6 diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-up.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out.yaml similarity index 100% rename from tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-up.yaml rename to tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-scale-out.yaml diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-cluster.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-status-00.yaml similarity index 85% rename from tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-cluster.yaml rename to tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-status-00.yaml index bc823bbb0..3d3d27ef6 100644 --- a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-cluster.yaml +++ b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-status-00.yaml @@ -5,5 +5,6 @@ metadata: name: redis-cluster-v1beta2 status: state: Ready + reason: RedisCluster is ready readyLeaderReplicas: 3 readyFollowerReplicas: 3 diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-sts-scale-up.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-sts-scale-up.yaml deleted file mode 100644 index 6de6ae80b..000000000 --- a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-sts-scale-up.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: redis-cluster-v1beta2-leader - labels: - app: redis-cluster-v1beta2-leader - redis_setup_type: cluster - role: leader -status: - replicas: 6 - readyReplicas: 6 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: redis-cluster-v1beta2-follower - labels: - app: redis-cluster-v1beta2-follower - redis_setup_type: cluster - role: follower -status: - replicas: 6 - readyReplicas: 6 diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-sts.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-sts.yaml deleted file mode 100644 index 73545eaab..000000000 --- a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/ready-sts.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: redis-cluster-v1beta2-leader - labels: - app: redis-cluster-v1beta2-leader - redis_setup_type: cluster - role: leader -status: - replicas: 3 - readyReplicas: 3 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: redis-cluster-v1beta2-follower - labels: - app: redis-cluster-v1beta2-follower - redis_setup_type: cluster - role: follower -status: - replicas: 3 - readyReplicas: 3