-
Notifications
You must be signed in to change notification settings - Fork 46
/
restore-cassandra.groovy
179 lines (161 loc) · 9.99 KB
/
restore-cassandra.groovy
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
/**
* Update packages on given nodes
*
* Expected parameters:
* SALT_MASTER_CREDENTIALS Credentials to the Salt API.
* SALT_MASTER_URL Full Salt API address [http://10.10.10.1:8000].
*
**/
common = new com.mirantis.mk.Common()
salt = new com.mirantis.mk.Salt()
python = new com.mirantis.mk.Python()
def pepperEnv = "pepperEnv"
def getValueForPillarKey(pepperEnv, target, pillarKey) {
def out = salt.getReturnValues(salt.getPillar(pepperEnv, target, pillarKey))
if (out == '') {
throw new Exception("Cannot get value for ${pillarKey} key on ${target} target")
}
return out.toString()
}
timeout(time: 12, unit: 'HOURS') {
node() {
stage('Setup virtualenv for Pepper') {
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
stage('Opencontrail controllers health check') {
try {
salt.enforceState(pepperEnv, 'I@opencontrail:control or I@opencontrail:collector', 'opencontrail.upgrade.verify', true, true)
} catch (Exception er) {
common.errorMsg("Opencontrail controllers health check stage found issues with currently running services.")
}
}
stage('Restore') {
// stop neutron-server to prevent CRUD api calls to contrail-api service
try {
salt.runSaltProcessStep(pepperEnv, 'I@neutron:server', 'service.stop', ['neutron-server'], null, true)
} catch (Exception er) {
common.warningMsg('neutron-server service already stopped')
}
// get opencontrail version
def contrailVersion = getValueForPillarKey(pepperEnv, "I@opencontrail:control:role:primary", "_param:opencontrail_version")
def configDbIp = getValueForPillarKey(pepperEnv, "I@opencontrail:control:role:primary", "opencontrail:database:bind:host")
def configDbPort = getValueForPillarKey(pepperEnv, "I@opencontrail:control:role:primary", "opencontrail:database:bind:port_configdb")
common.infoMsg("OpenContrail version is ${contrailVersion}")
if (contrailVersion.startsWith('4')) {
controllerImage = getValueForPillarKey(pepperEnv, "I@opencontrail:control:role:primary",
"docker:client:compose:opencontrail:service:controller:container_name")
common.infoMsg("Applying db restore procedure for OpenContrail 4.X version")
try {
common.infoMsg("Stop contrail control plane containers")
salt.cmdRun(pepperEnv, 'I@opencontrail:control or I@opencontrail:collector', 'cd /etc/docker/compose/opencontrail/; docker-compose down')
} catch (Exception err) {
common.errorMsg('An error has been occurred during contrail containers shutdown: ' + err.getMessage())
throw err
}
try {
common.infoMsg("Cleanup cassandra data")
salt.cmdRun(pepperEnv, 'I@opencontrail:control', 'for f in $(ls /var/lib/configdb/); do rm -r /var/lib/configdb/$f; done')
} catch (Exception err) {
common.errorMsg('Cannot cleanup cassandra data on control nodes: ' + err.getMessage())
throw err
}
try {
common.infoMsg("Start cassandra db on I@cassandra:backup:client node")
salt.cmdRun(pepperEnv, 'I@cassandra:backup:client', 'cd /etc/docker/compose/opencontrail/; docker-compose up -d')
} catch (Exception err) {
common.errorMsg('An error has been occurred during cassandra db startup on I@cassandra:backup:client node: ' + err.getMessage())
throw err
}
// wait for cassandra to be online
common.retry(6, 20){
common.infoMsg("Trying to connect to casandra db on I@cassandra:backup:client node ...")
salt.cmdRun(pepperEnv, 'I@cassandra:backup:client', "nc -v -z -w2 ${configDbIp} ${configDbPort}")
}
// remove restore-already-happened file if any is present
try {
salt.cmdRun(pepperEnv, 'I@cassandra:backup:client', 'rm /var/backups/cassandra/dbrestored')
} catch (Exception err) {
common.warningMsg('/var/backups/cassandra/dbrestored not present? ' + err.getMessage())
}
salt.enforceState(pepperEnv, 'I@cassandra:backup:client', "cassandra")
try {
salt.cmdRun(pepperEnv, 'I@opencontrail:control and not I@cassandra:backup:client', 'cd /etc/docker/compose/opencontrail/; docker-compose up -d')
} catch (Exception err) {
common.errorMsg('An error has been occurred during cassandra db startup on I@opencontrail:control and not I@cassandra:backup:client nodes: ' + err.getMessage())
throw err
}
// wait till outstanding cassandra dbs are up
common.retry(6, 20){
common.infoMsg("Trying to connect to casandra db on I@opencontrail:control and not I@cassandra:backup:client nodes ...")
salt.cmdRun(pepperEnv, 'I@opencontrail:control and not I@cassandra:backup:client', "nc -v -z -w2 ${configDbIp} ${configDbPort}")
}
try {
common.infoMsg("Start analytics containers node")
salt.cmdRun(pepperEnv, 'I@opencontrail:collector', 'cd /etc/docker/compose/opencontrail/; docker-compose up -d')
} catch (Exception err) {
common.errorMsg('An error has been occurred during analytics containers startup: ' + err.getMessage())
throw err
}
// contrail-control service needs to be restart after db sync to re-initialize with recovered data
common.retry(6,20) {
common.infoMsg("Restart contrail-control services on control nodes")
salt.cmdRun(pepperEnv, 'I@opencontrail:control', 'doctrail controller service contrail-control restart')
}
} else {
try {
salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'service.stop', ['supervisor-config'], null, true)
} catch (Exception er) {
common.warningMsg('Supervisor-config service already stopped')
}
// Cassandra restore section
try {
salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'service.stop', ['supervisor-database'], null, true)
} catch (Exception er) {
common.warningMsg('Supervisor-database service already stopped')
}
try {
salt.cmdRun(pepperEnv, 'I@opencontrail:control', "mkdir -p /root/cassandra/cassandra.bak")
} catch (Exception er) {
common.warningMsg('Directory already exists')
}
try {
salt.cmdRun(pepperEnv, 'I@opencontrail:control', "mv /var/lib/cassandra/* /root/cassandra/cassandra.bak")
} catch (Exception er) {
common.warningMsg('Files were already moved')
}
try {
salt.cmdRun(pepperEnv, 'I@opencontrail:control', "rm -rf /var/lib/cassandra/*")
} catch (Exception er) {
common.warningMsg('Directory already empty')
}
def backupDir = getValueForPillarKey(pepperEnv, "I@cassandra:backup:client", "cassandra:backup:backup_dir")
common.infoMsg("Backup directory is ${backupDir}")
salt.runSaltProcessStep(pepperEnv, 'I@cassandra:backup:client', 'file.remove', ["${backupDir}/dbrestored"], null, true)
salt.runSaltProcessStep(pepperEnv, 'I@cassandra:backup:client', 'service.start', ['supervisor-database'], null, true)
// wait until supervisor-database service is up
salt.commandStatus(pepperEnv, 'I@cassandra:backup:client', 'service supervisor-database status', 'running')
sleep(60)
// performs restore
salt.enforceState(pepperEnv, 'I@cassandra:backup:client', "cassandra.backup")
salt.runSaltProcessStep(pepperEnv, 'I@cassandra:backup:client', 'system.reboot', null, null, true, 5)
sleep(5)
salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control and not I@cassandra:backup:client', 'system.reboot', null, null, true, 5)
// wait until supervisor-database service is up
salt.commandStatus(pepperEnv, 'I@cassandra:backup:client', 'service supervisor-database status', 'running')
salt.commandStatus(pepperEnv, 'I@opencontrail:control and not I@cassandra:backup:client', 'service supervisor-database status', 'running')
sleep(5)
salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'service.restart', ['supervisor-database'], null, true)
// wait until contrail-status is up
salt.commandStatus(pepperEnv, 'I@opencontrail:control', "contrail-status | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup", null, false)
salt.cmdRun(pepperEnv, 'I@opencontrail:control', "nodetool status")
salt.cmdRun(pepperEnv, 'I@opencontrail:control', "contrail-status")
}
salt.runSaltProcessStep(pepperEnv, 'I@neutron:server', 'service.start', ['neutron-server'], null, true)
}
stage('Opencontrail controllers health check') {
common.retry(9, 60){
salt.enforceState(pepperEnv, 'I@opencontrail:control or I@opencontrail:collector', 'opencontrail.upgrade.verify', true, true)
}
}
}
}