Skip to content

Commit

Permalink
external-replication: stop&start fixes (#42)
Browse files Browse the repository at this point in the history 
* external-replication: stop&start fixes

* external-replication: stop&start fixes

* external-replication: stop&start fixes

* external-replication: stop&start fixes

* external-replication: stop&start fixes

---------

Co-authored-by: suetin <[email protected]>
  • Loading branch information
@moridin26
moridin26 and suetin authored Sep 18, 2023
1 parent 996df53 commit f7d5bbb
Show file tree
Hide file tree
Showing 5 changed files with 261 additions and 14 deletions.
2 changes: 1 addition & 1 deletion internal/app/app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1802,7 +1802,7 @@ func (app *App) repairExternalReplication(masterNode *mysql.Node) {
return
}

if extReplStatus.ReplicationState() == mysql.ReplicationError {
if masterNode.IsExternalReplicationRunningByUser() && !extReplStatus.ReplicationRunning() {
// TODO: remove "". Master is not needed for external replication now
app.TryRepairReplication(masterNode, "", app.config.ExternalReplicationChannel)
}
Expand Down
28 changes: 21 additions & 7 deletions internal/mysql/data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ const (
ReplicationError = "error"
)

const (
ReplicationRunningByClient = "running"
ReplicationStoppedByClient = "stopped"
)

type pingResult struct {
Ok int `db:"Ok"`
}
Expand Down Expand Up @@ -41,13 +46,14 @@ type ResetupStatus struct {
}

type replicationSettings struct {
ChannelName string `db:"ChannelName"`
SourceHost string `db:"SourceHost"`
SourceUser string `db:"SourceUser"`
SourcePassword string `db:"SourcePassword"`
SourcePort int `db:"SourcePort"`
SourceSslCa string `db:"SourceSslCa"`
SourceDelay int `db:"SourceDelay"`
ChannelName string `db:"ChannelName"`
SourceHost string `db:"SourceHost"`
SourceUser string `db:"SourceUser"`
SourcePassword string `db:"SourcePassword"`
SourcePort int `db:"SourcePort"`
SourceSslCa string `db:"SourceSslCa"`
SourceDelay int `db:"SourceDelay"`
ReplicationStatus sql.NullString `db:"ReplicationStatus"`
}

// SlaveStatusStruct contains SHOW SLAVE STATUS response
Expand Down Expand Up @@ -102,6 +108,14 @@ type SemiSyncStatus struct {
WaitSlaveCount int `db:"WaitSlaveCount"`
}

func (sett *replicationSettings) ShouldBeRunning() bool {
replStatus, _ := sett.ReplicationStatus.Value()
if replStatus != nil {
return replStatus == ReplicationRunningByClient
}
return false
}

func (ss *SlaveStatusStruct) GetMasterHost() string {
return ss.MasterHost
}
Expand Down
17 changes: 16 additions & 1 deletion internal/mysql/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1004,7 +1004,22 @@ func (n *Node) SetExternalReplication() error {
if err != nil {
return err
}
return n.StartExternalReplication()
if replSettings.ShouldBeRunning() {
return n.StartExternalReplication()
}
return nil
}

func (n *Node) IsExternalReplicationRunningByUser() bool {
var replSettings replicationSettings
err := n.queryRow(queryGetExternalReplicationSettings, nil, &replSettings)
if err != nil {
return false
}
if replSettings.ShouldBeRunning() {
return true
}
return false
}

func (n *Node) UpdateExternalCAFile() error {
Expand Down
2 changes: 1 addition & 1 deletion internal/mysql/queries.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ var DefaultQueries = map[string]string{
queryHasWaitingSemiSyncAck: `SELECT count(*) <> 0 AS IsWaiting FROM information_schema.PROCESSLIST WHERE state = 'Waiting for semi-sync ACK from slave'`,
queryGetLastStartupTime: `SELECT UNIX_TIMESTAMP(DATE_SUB(now(), INTERVAL variable_value SECOND)) AS LastStartup FROM performance_schema.global_status WHERE variable_name='Uptime'`,
queryGetExternalReplicationSettings: `SELECT channel_name AS ChannelName, source_host AS SourceHost, source_user AS SourceUser, source_port AS SourcePort,
source_password AS SourcePassword, source_ssl_ca AS SourceSslCa, source_delay AS SourceDelay
source_password AS SourcePassword, source_ssl_ca AS SourceSslCa, source_delay AS SourceDelay, replication_status AS ReplicationStatus
FROM mysql.replication_settings WHERE channel_name = 'external'`,
queryChangeSource: `CHANGE REPLICATION SOURCE TO
SOURCE_HOST = :host,
Expand Down
226 changes: 222 additions & 4 deletions tests/features/external_replication.feature
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ Feature: external replication
And I run SQL on mysql host "mysql1"
"""
INSERT INTO mysql.replication_settings
(channel_name, source_host, source_user, source_password, source_port)
VALUES ('external', 'test_source_2', 'test_user_2', 'test_pass_2', 2222);
(channel_name, source_host, source_user, source_password, source_port, replication_status)
VALUES ('external', 'test_source_2', 'test_user_2', 'test_pass_2', 2222, 'running');
"""
And I run SQL on mysql host "mysql1" expecting error on number "3074"
"""
Expand Down Expand Up @@ -88,6 +88,7 @@ Feature: external replication
[{
"Exec_Source_Log_Pos": "0",
"Replica_IO_State": "Connecting to source",
"Replica_SQL_Running": "Yes",
"Source_Host": "test_source",
"Source_Port": "1111",
"Source_User": "test_user",
Expand Down Expand Up @@ -184,6 +185,7 @@ YZQy1bHIhscLf8wjTYbzAg==
"Source_Port": "2222",
"Source_User": "test_user_2",
"Replica_IO_Running": "Connecting",
"Replica_SQL_Running": "Yes",
"Relay_Source_Log_File": "",
"Exec_Source_Log_Pos": "0",
"Channel_Name": "external",
Expand Down Expand Up @@ -238,8 +240,8 @@ YZQy1bHIhscLf8wjTYbzAg==
And I run SQL on mysql host "mysql1"
"""
INSERT INTO mysql.replication_settings
(channel_name, source_host, source_user, source_password, source_port)
VALUES ('external', 'test_source', 'test_user', 'test_pass', 2222)
(channel_name, source_host, source_user, source_password, source_port, replication_status)
VALUES ('external', 'test_source', 'test_user', 'test_pass', 2222, 'stopped')
"""
And I run SQL on mysql host "mysql1"
"""
Expand All @@ -261,6 +263,27 @@ YZQy1bHIhscLf8wjTYbzAg==
"Source_Port": "1111",
"Source_User": "test_user",
"Replica_IO_Running": "No",
"Replica_SQL_Running": "No",
"Source_SSL_CA_File": "",
"Relay_Source_Log_File": "",
"Exec_Source_Log_Pos": "0",
"Channel_Name": "external"
}]
"""
When I wait for "10" seconds
And I run SQL on mysql host "mysql1"
"""
SHOW REPLICA STATUS FOR CHANNEL 'external'
"""
Then SQL result should match json
"""
[{
"Replica_IO_State": "",
"Source_Host": "test_source",
"Source_Port": "1111",
"Source_User": "test_user",
"Replica_IO_Running": "No",
"Replica_SQL_Running": "No",
"Source_SSL_CA_File": "",
"Relay_Source_Log_File": "",
"Exec_Source_Log_Pos": "0",
Expand Down Expand Up @@ -304,6 +327,7 @@ YZQy1bHIhscLf8wjTYbzAg==
"Source_Port": "1111",
"Source_User": "test_user",
"Replica_IO_Running": "No",
"Replica_SQL_Running": "No",
"Source_SSL_CA_File": "",
"Relay_Source_Log_File": "",
"Exec_Source_Log_Pos": "0",
Expand All @@ -329,6 +353,7 @@ YZQy1bHIhscLf8wjTYbzAg==
"Source_Port": "1111",
"Source_User": "test_user",
"Replica_IO_Running": "No",
"Replica_SQL_Running": "No",
"Source_SSL_CA_File": "",
"Relay_Source_Log_File": "",
"Exec_Source_Log_Pos": "0",
Expand Down Expand Up @@ -372,9 +397,202 @@ Y2AirKuDzA5GErKOfQ==
"Source_Port": "1111",
"Source_User": "test_user",
"Replica_IO_Running": "No",
"Replica_SQL_Running": "No",
"Source_SSL_CA_File": "",
"Relay_Source_Log_File": "",
"Exec_Source_Log_Pos": "0",
"Channel_Name": "external"
}]
"""
When I run SQL on mysql host "mysql1"
"""
UPDATE mysql.replication_settings SET replication_status = 'running' WHERE channel_name = 'external'
"""
And I wait for "70" seconds
And I run SQL on mysql host "mysql1"
"""
SHOW REPLICA STATUS FOR CHANNEL 'external'
"""
Then SQL result should match json
"""
[{
"Replica_IO_State": "Connecting to source",
"Source_Host": "test_source",
"Source_Port": "1111",
"Source_User": "test_user",
"Replica_IO_Running": "Connecting",
"Replica_SQL_Running": "Yes",
"Source_SSL_CA_File": "",
"Relay_Source_Log_File": "",
"Exec_Source_Log_Pos": "0",
"Channel_Name": "external"
}]
"""

Scenario: external replication stop/start by user
Given cluster is up and running
Then mysql host "mysql1" should be master
And mysql host "mysql2" should be replica of "mysql1"
And mysql host "mysql3" should be replica of "mysql1"
When I run SQL on mysql host "mysql1"
"""
CREATE TABLE mysql.replication_settings(
channel_name VARCHAR(50) NOT NULL,
source_host VARCHAR(50) NOT NULL,
source_user VARCHAR(50) NOT NULL,
source_password VARCHAR(50) NOT NULL,
source_port INT UNSIGNED NOT NULL,
source_ssl_ca VARCHAR(4096) NOT NULL DEFAULT '',
source_delay INT UNSIGNED NOT NULL DEFAULT 0,
source_log_file VARCHAR(50) NOT NULL DEFAULT '',
source_log_pos INT UNSIGNED NOT NULL DEFAULT 0,
replication_status ENUM ('stopped', 'running') NOT NULL DEFAULT 'stopped',
PRIMARY KEY (channel_name)
) ENGINE=INNODB;
"""
And I run SQL on mysql host "mysql1"
"""
INSERT INTO mysql.replication_settings
(channel_name, source_host, source_user, source_password, source_port, replication_status)
VALUES ('external', 'test_source_2', 'test_user_2', 'test_pass_2', 2222, 'stopped');
"""
And I run SQL on mysql host "mysql1" expecting error on number "3074"
"""
SHOW REPLICA STATUS FOR CHANNEL 'external'
"""
And I run SQL on mysql host "mysql1"
"""
SELECT source_host, source_user, source_password, source_port FROM mysql.replication_settings WHERE channel_name = 'external'
"""
Then SQL result should match json
"""
[{
"source_host": "test_source_2",
"source_password": "test_pass_2",
"source_port": "2222",
"source_user": "test_user_2"
}]
"""
When I wait for "5" seconds
And I run SQL on mysql host "mysql2"
"""
SELECT source_host, source_user, source_password, source_port FROM mysql.replication_settings WHERE channel_name = 'external'
"""
Then SQL result should match json
"""
[{
"source_host": "test_source_2",
"source_port": "2222",
"source_password": "test_pass_2",
"source_user": "test_user_2"
}]
"""
And I run SQL on mysql host "mysql1" expecting error on number "3074"
"""
SHOW REPLICA STATUS FOR CHANNEL 'external'
"""
When I run SQL on mysql host "mysql1"
"""
CHANGE REPLICATION SOURCE TO SOURCE_HOST = 'test_source',
SOURCE_USER = 'test_user',
SOURCE_PASSWORD = 'test_pass',
SOURCE_PORT = 1111,
SOURCE_AUTO_POSITION = 1
FOR CHANNEL 'external'
"""
And I run SQL on mysql host "mysql2" expecting error on number "3074"
"""
SHOW REPLICA STATUS FOR CHANNEL 'external'
"""
And I run SQL on mysql host "mysql1"
"""
START REPLICA FOR CHANNEL 'external'
"""
And I run SQL on mysql host "mysql1"
"""
SHOW REPLICA STATUS FOR CHANNEL 'external'
"""
Then SQL result should match json
"""
[{
"Exec_Source_Log_Pos": "0",
"Replica_IO_State": "Connecting to source",
"Source_Host": "test_source",
"Source_Port": "1111",
"Source_User": "test_user",
"Replica_IO_Running": "Connecting",
"Replica_SQL_Running": "Yes",
"Relay_Source_Log_File": "",
"Channel_Name": "external",
"Source_SSL_CA_File": ""
}]
"""
When I run command on host "mysql1"
"""
mysync switch --to mysql2 --wait=0s
"""
Then command return code should be "0"
And command output should match regexp
"""
switchover scheduled
"""
And zookeeper node "/test/switch" should match json
"""
{
"from": "",
"to": "mysql2"
}
"""
Then zookeeper node "/test/last_switch" should match json within "30" seconds
"""
{
"from": "",
"to": "mysql2",
"result": {
"ok": true
}
}
"""
Then mysql host "mysql2" should be master
And mysql host "mysql2" should be writable
When I run SQL on mysql host "mysql2"
"""
SHOW REPLICA STATUS FOR CHANNEL 'external'
"""
Then SQL result should match json
"""
[{
"Replica_IO_State": "",
"Source_Host": "test_source_2",
"Source_Port": "2222",
"Source_User": "test_user_2",
"Replica_IO_Running": "No",
"Replica_SQL_Running": "No",
"Relay_Source_Log_File": "",
"Exec_Source_Log_Pos": "0",
"Channel_Name": "external",
"Replicate_Ignore_DB": "mysql",
"Source_SSL_CA_File": ""
}]
"""
When host "mysql1" is started
Then mysql host "mysql1" should become available within "20" seconds
And mysql host "mysql1" should become replica of "mysql2" within "10" seconds
And I run SQL on mysql host "mysql1" expecting error on number "3074"
"""
SHOW REPLICA STATUS FOR CHANNEL 'external'
"""
Then I run SQL on mysql host "mysql1"
"""
SELECT source_host, source_user, source_password, source_port FROM mysql.replication_settings WHERE channel_name = 'external'
"""
Then SQL result should match json
"""
[{
"source_host": "test_source_2",
"source_user": "test_user_2",
"source_password": "test_pass_2",
"source_port": "2222"
}]
"""

0 comments on commit f7d5bbb

Please sign in to comment.