Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

K8SPS-359: Fix rebooting cluster if it's partially online #725

Merged
merged 4 commits into from
Aug 19, 2024
Merged

K8SPS-359: Fix rebooting cluster if it's partially online #725

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6bb9ea5
K8SPS-359: Fix rebooting cluster if it's partially online
egegunes Aug 13, 2024
d4a8e16
fix delete all pods
egegunes Aug 14, 2024
21ade91
Merge branch 'main' into K8SPS-359
egegunes Aug 19, 2024
75bb02d
fix cw-rbac
egegunes Aug 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions cmd/bootstrap/group_replication.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,19 @@ func bootstrapGroupReplication(ctx context.Context) error {

log.Printf("Cluster status:\n%s", status)

for _, member := range status.DefaultReplicaSet.Topology {
if member.MemberRole == innodbcluster.MemberRolePrimary && member.MemberState != innodbcluster.MemberStateOnline {
log.Printf("Primary (%s) is not ONLINE. Starting full cluster crash recovery...", member.Address)

if err := handleFullClusterCrash(ctx); err != nil {
return errors.Wrap(err, "handle full cluster crash")
}

// force restart container
os.Exit(1)
}
}

member, ok := status.DefaultReplicaSet.Topology[fmt.Sprintf("%s:%d", localShell.host, 3306)]
if !ok {
log.Printf("Adding instance (%s) to InnoDB cluster", localShell.host)
Expand Down
4 changes: 3 additions & 1 deletion cmd/bootstrap/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package main

import (
"context"
"io"
"log"
"os"
"path/filepath"
Expand All @@ -20,7 +21,8 @@ func main() {
log.Fatalf("error opening file: %v", err)
}
defer f.Close()
log.SetOutput(f)

log.SetOutput(io.MultiWriter(os.Stderr, f))

fullClusterCrash, err := fileExists(fullClusterCrashFile)
if err == nil && fullClusterCrash {
Expand Down
14 changes: 13 additions & 1 deletion config/rbac/cluster/role.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,21 @@ rules:
- apiGroups:
- ""
resources:
- configmaps
- pods
- pods/exec
verbs:
- create
- delete
- deletecollection
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- configmaps
- secrets
- services
verbs:
Expand Down
16 changes: 14 additions & 2 deletions config/rbac/role.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@ rules:
resources:
- configmaps
- persistentvolumeclaims
- pods
- pods/exec
- secrets
- services
verbs:
Expand All @@ -28,6 +26,20 @@ rules:
verbs:
- create
- patch
- apiGroups:
- ""
resources:
- pods
- pods/exec
verbs:
- create
- delete
- deletecollection
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
Expand Down
16 changes: 14 additions & 2 deletions deploy/bundle.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11437,8 +11437,6 @@ rules:
resources:
- configmaps
- persistentvolumeclaims
- pods
- pods/exec
- secrets
- services
verbs:
Expand All @@ -11456,6 +11454,20 @@ rules:
verbs:
- create
- patch
- apiGroups:
- ""
resources:
- pods
- pods/exec
verbs:
- create
- delete
- deletecollection
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
Expand Down
14 changes: 13 additions & 1 deletion deploy/cw-bundle.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11445,9 +11445,21 @@ rules:
- apiGroups:
- ""
resources:
- configmaps
- pods
- pods/exec
verbs:
- create
- delete
- deletecollection
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- configmaps
- secrets
- services
verbs:
Expand Down
14 changes: 13 additions & 1 deletion deploy/cw-rbac.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,21 @@ rules:
- apiGroups:
- ""
resources:
- configmaps
- pods
- pods/exec
verbs:
- create
- delete
- deletecollection
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- configmaps
- secrets
- services
verbs:
Expand Down
16 changes: 14 additions & 2 deletions deploy/rbac.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,6 @@ rules:
resources:
- configmaps
- persistentvolumeclaims
- pods
- pods/exec
- secrets
- services
verbs:
Expand All @@ -69,6 +67,20 @@ rules:
verbs:
- create
- patch
- apiGroups:
- ""
resources:
- pods
- pods/exec
verbs:
- create
- delete
- deletecollection
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
Expand Down
5 changes: 4 additions & 1 deletion e2e-tests/functions
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -575,7 +575,10 @@ get_primary_from_haproxy() {
}

get_primary_from_group_replication() {
run_mysql "SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='PRIMARY';" "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" | cut -d'.' -f1
run_mysql \
"SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='PRIMARY';" \
"-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" \
| cut -d'.' -f1
Comment on lines +579 to +581
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[shfmt] reported by reviewdog 🐶

Suggested change
"SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='PRIMARY';" \
"-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" \
| cut -d'.' -f1
"SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='PRIMARY';" \
"-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" \
| cut -d'.' -f1

}

verify_certificate_sans() {
Expand Down
2 changes: 1 addition & 1 deletion e2e-tests/tests/gr-self-healing/01-deploy-chaos-mesh.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
apiVersion: kuttl.dev/v1beta1
kind: TestStep
timeout: 10
commands:
- script: |-
set -o errexit
Expand All @@ -9,3 +8,4 @@ commands:
source ../../functions

deploy_chaos_mesh
timeout: 120
2 changes: 1 addition & 1 deletion e2e-tests/tests/gr-self-healing/14-cluster-crash.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
apiVersion: kuttl.dev/v1beta1
kind: TestStep
timeout: 30
commands:
- script: |-
set -o errexit
Expand All @@ -10,3 +9,4 @@ commands:

kill_pods "${NAMESPACE}" "label" "app.kubernetes.io/instance" "gr-self-healing" "cluster-crash"
sleep 30 # wait for crash
timeout: 40
Loading
Loading