diff --git a/README.md b/README.md index 2b9e6159..7619bf50 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ replication_repair_max_attempts: 3 external_replication_type: off show_only_gtid_diff: False +force_switchover: False ``` ### Usage diff --git a/internal/app/app.go b/internal/app/app.go index 02f79fe4..398619a7 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -1208,6 +1208,21 @@ func (app *App) performSwitchover(clusterState map[string]*NodeState, activeNode } node := app.cluster.Get(host) // in case node is a master + + if app.config.ForceSwitchover { + err := node.SetOfflineForce() + if err != nil { + return fmt.Errorf("failed to set node %s force offline: %v", host, err) + } + + defer func() { + err := node.SetOnline() + if err != nil { + app.logger.Errorf("failed to set node %s online after setting force offline: %v", host, err) + } + }() + } + err := node.SetReadOnly(true) if err != nil || app.emulateError("freeze_ro") { app.logger.Infof("switchover: failed to set node %s read-only, trying kill bad queries: %v", host, err) diff --git a/internal/config/config.go b/internal/config/config.go index 21b00332..88837a7b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -97,6 +97,7 @@ type Config struct { ReplMonErrorWaitInterval time.Duration `config:"repl_mon_error_wait_interval" yaml:"repl_mon_error_wait_interval"` ReplMonSlaveWaitInterval time.Duration `config:"repl_mon_slave_wait_interval" yaml:"repl_mon_slave_wait_interval"` ShowOnlyGTIDDiff bool `config:"show_only_gtid_diff" yaml:"show_only_gtid_diff"` + ForceSwitchover bool `config:"force_switchover" yaml:"force_switchover"` // TODO: Remove when we will be sure it's right way to do switchover } // DefaultConfig returns default configuration for MySync @@ -182,6 +183,7 @@ func DefaultConfig() (Config, error) { ReplMonErrorWaitInterval: 10 * time.Second, ReplMonSlaveWaitInterval: 10 * time.Second, ShowOnlyGTIDDiff: false, + ForceSwitchover: false, } return config, nil } diff --git a/internal/mysql/node.go b/internal/mysql/node.go index 4475e56a..ef9dd742 100644 --- a/internal/mysql/node.go +++ b/internal/mysql/node.go @@ -833,6 +833,14 @@ func (n *Node) SetOnline() error { return n.exec(queryDisableOfflineMode, nil) } +func (n *Node) SetOfflineForce() error { + err := n.SemiSyncDisable() + if err != nil { + return err + } + return n.SetOffline() +} + // ChangeMaster changes master of MySQL Node, demoting it to slave func (n *Node) ChangeMaster(host string) error { useSsl := 0 diff --git a/tests/features/switchover_from.feature b/tests/features/switchover_from.feature index c83bfa18..1686d1b0 100644 --- a/tests/features/switchover_from.feature +++ b/tests/features/switchover_from.feature @@ -37,7 +37,11 @@ Feature: manual switchover from old master } """ - Scenario: if switchover was approved, it will not be rejected + Scenario Outline: if switchover was approved, it will not be rejected + Given cluster environment is + """ + FORCE_SWITCHOVER= + """ Given cluster is up and running Then zookeeper node "/test/active_nodes" should match json_exactly within "20" seconds """ @@ -71,6 +75,10 @@ Feature: manual switchover from old master } } """ + Examples: + | force_switchover | + | true | + | false | Scenario Outline: switchover from works on healthy cluster Given cluster environment is diff --git a/tests/features/switchover_to.feature b/tests/features/switchover_to.feature index ebacfe50..16a8f01d 100644 --- a/tests/features/switchover_to.feature +++ b/tests/features/switchover_to.feature @@ -1,7 +1,11 @@ Feature: manual switchover to new master - Scenario: switchover on kill all running query on old master + Scenario Outline: switchover on kill all running query on old master + Given cluster environment is + """ + FORCE_SWITCHOVER= + """ Given cluster is up and running Then mysql host "mysql1" should be master And mysql host "mysql2" should be replica of "mysql1" @@ -54,6 +58,10 @@ Feature: manual switchover to new master And mysql host "mysql3" should have variable "rpl_semi_sync_master_enabled" set to "0" And mysql replication on host "mysql3" should run fine within "3" seconds And mysql host "mysql3" should be read only + Examples: + | force_switchover | + | true | + | false | Scenario Outline: switchover to works on healthy cluster Given cluster environment is @@ -274,4 +282,3 @@ Feature: manual switchover to new master """ mysql2 is not active """ - diff --git a/tests/images/mysql/mysync.yaml b/tests/images/mysql/mysync.yaml index a2a47a3d..d7a6a9a5 100644 --- a/tests/images/mysql/mysync.yaml +++ b/tests/images/mysql/mysync.yaml @@ -62,3 +62,4 @@ replication_channel: '' external_replication_type: 'external' show_only_gtid_diff: false repl_mon: ${REPL_MON:-false} +force_switchover: ${FORCE_SWITCHOVER:-false}