Merge branch 'main' into K8SPS-367-delete-pvc-finalizer

percona · Aug 20, 2024 · 8397b7e · 8397b7e
2 parents c8dca98 + d17dc59
commit 8397b7e
Show file tree

Hide file tree

Showing 17 changed files with 352 additions and 14 deletions.
diff --git a/cmd/bootstrap/group_replication.go b/cmd/bootstrap/group_replication.go
@@ -373,6 +373,19 @@ func bootstrapGroupReplication(ctx context.Context) error {
 
 	log.Printf("Cluster status:\n%s", status)
 
+	for _, member := range status.DefaultReplicaSet.Topology {
+		if member.MemberRole == innodbcluster.MemberRolePrimary && member.MemberState != innodbcluster.MemberStateOnline {
+			log.Printf("Primary (%s) is not ONLINE. Starting full cluster crash recovery...", member.Address)
+
+			if err := handleFullClusterCrash(ctx); err != nil {
+				return errors.Wrap(err, "handle full cluster crash")
+			}
+
+			// force restart container
+			os.Exit(1)
+		}
+	}
+
 	member, ok := status.DefaultReplicaSet.Topology[fmt.Sprintf("%s:%d", localShell.host, 3306)]
 	if !ok {
 		log.Printf("Adding instance (%s) to InnoDB cluster", localShell.host)

diff --git a/cmd/bootstrap/main.go b/cmd/bootstrap/main.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"io"
 	"log"
 	"os"
 	"path/filepath"
@@ -20,7 +21,8 @@ func main() {
 		log.Fatalf("error opening file: %v", err)
 	}
 	defer f.Close()
-	log.SetOutput(f)
+
+	log.SetOutput(io.MultiWriter(os.Stderr, f))
 
 	fullClusterCrash, err := fileExists(fullClusterCrashFile)
 	if err == nil && fullClusterCrash {

diff --git a/config/rbac/cluster/role.yaml b/config/rbac/cluster/role.yaml
@@ -17,9 +17,21 @@ rules:
 - apiGroups:
   - ""
   resources:
-  - configmaps
   - pods
   - pods/exec
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
   - secrets
   - services
   verbs:

diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -9,8 +9,6 @@ rules:
   resources:
   - configmaps
   - persistentvolumeclaims
-  - pods
-  - pods/exec
   - secrets
   - services
   verbs:
@@ -28,6 +26,20 @@ rules:
   verbs:
   - create
   - patch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - pods/exec
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:

diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml
@@ -11437,8 +11437,6 @@ rules:
   resources:
   - configmaps
   - persistentvolumeclaims
-  - pods
-  - pods/exec
   - secrets
   - services
   verbs:
@@ -11456,6 +11454,20 @@ rules:
   verbs:
   - create
   - patch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - pods/exec
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:

diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml
@@ -11445,9 +11445,21 @@ rules:
 - apiGroups:
   - ""
   resources:
-  - configmaps
   - pods
   - pods/exec
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
   - secrets
   - services
   verbs:

diff --git a/deploy/cw-rbac.yaml b/deploy/cw-rbac.yaml
@@ -58,9 +58,21 @@ rules:
 - apiGroups:
   - ""
   resources:
-  - configmaps
   - pods
   - pods/exec
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
   - secrets
   - services
   verbs:

diff --git a/deploy/rbac.yaml b/deploy/rbac.yaml
@@ -50,8 +50,6 @@ rules:
   resources:
   - configmaps
   - persistentvolumeclaims
-  - pods
-  - pods/exec
   - secrets
   - services
   verbs:
@@ -69,6 +67,20 @@ rules:
   verbs:
   - create
   - patch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - pods/exec
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:

diff --git a/e2e-tests/functions b/e2e-tests/functions
@@ -575,7 +575,10 @@ get_primary_from_haproxy() {
 }
 
 get_primary_from_group_replication() {
-	run_mysql "SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='PRIMARY';" "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" | cut -d'.' -f1
+	run_mysql \
+    "SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='PRIMARY';" \
+    "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" \
+      | cut -d'.' -f1
 }
 
 verify_certificate_sans() {

diff --git a/e2e-tests/tests/gr-self-healing/01-deploy-chaos-mesh.yaml b/e2e-tests/tests/gr-self-healing/01-deploy-chaos-mesh.yaml
@@ -1,6 +1,5 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
-timeout: 10
 commands:
   - script: |-
       set -o errexit
@@ -9,3 +8,4 @@ commands:
       source ../../functions
 
       deploy_chaos_mesh
+    timeout: 120
diff --git a/e2e-tests/tests/gr-self-healing/14-cluster-crash.yaml b/e2e-tests/tests/gr-self-healing/14-cluster-crash.yaml
@@ -1,6 +1,5 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
-timeout: 30
 commands:
   - script: |-
       set -o errexit
@@ -10,3 +9,4 @@ commands:
 
       kill_pods "${NAMESPACE}" "label" "app.kubernetes.io/instance" "gr-self-healing" "cluster-crash"
       sleep 30 # wait for crash
+    timeout: 40