-
Notifications
You must be signed in to change notification settings - Fork 46
/
cvp-ha.groovy
213 lines (206 loc) · 11.2 KB
/
cvp-ha.groovy
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
/**
*
* Launch CVP HA testing for the cloud (virtualized control plane only)
*
* Expected parameters:
*
* SALT_MASTER_URL URL of Salt master
* SALT_MASTER_CREDENTIALS Credentials that are used in this Jenkins for accessing Salt master (usually "salt")
* PROXY Proxy address (if any) for accessing the Internet. It will be used for cloning repos and installing pip dependencies
* TEST_IMAGE Docker image link to use for running container with testing tools.
* TOOLS_REPO URL of repo where testing tools, scenarios, configs are located
*
* DEBUG_MODE If you need to debug (keep container after test), please enabled this
* MANUAL_CONFIRMATION Ask for confirmation before doing something destructive (reboot/shutdown node)
* RETRY_CHECK_STATUS Number of retries to check node status
* SKIP_LIST_PATH Path to tempest skip list file in TOOLS_REPO
* TARGET_NODES Nodes to test
* TEMPEST_REPO Tempest repo to clone and use
* TEMPEST_TARGET_NODE Node, where tests will be executed
* TEMPEST_TEST_PATTERN Tests to run during HA scenarios
*
*/
common = new com.mirantis.mk.Common()
salt = new com.mirantis.mk.Salt()
validate = new com.mirantis.mcp.Validate()
def saltMaster
def artifacts_dir = 'validation_artifacts/'
def remote_artifacts_dir = '/root/qa_results/'
def current_target_node = null
def first_node = null
def tempest_result = ''
timeout(time: 12, unit: 'HOURS') {
node() {
def num_retries = Integer.parseInt(RETRY_CHECK_STATUS)
try {
stage('Initialization') {
sh "rm -rf ${artifacts_dir}"
if (!TEMPEST_TARGET_NODE) {
// This pillar will return us cid01
TEMPEST_TARGET_NODE = "I@gerrit:client"
}
saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
os_version=salt.getPillar(saltMaster, 'I@salt:master', '_param:openstack_version')['return'][0].values()[0]
if (!os_version) {
throw new Exception("Openstack is not found on this env. Exiting")
}
salt.cmdRun(saltMaster, TEMPEST_TARGET_NODE, "rm -rf ${remote_artifacts_dir}")
salt.cmdRun(saltMaster, TEMPEST_TARGET_NODE, "mkdir -p ${remote_artifacts_dir}")
keystone_creds = validate._get_keystone_creds_v3(saltMaster)
if (!keystone_creds) {
keystone_creds = validate._get_keystone_creds_v2(saltMaster)
}
validate.runContainer(saltMaster, TEMPEST_TARGET_NODE, TEST_IMAGE, 'cvp', keystone_creds)
validate.configureContainer(saltMaster, TEMPEST_TARGET_NODE, PROXY, TOOLS_REPO, TEMPEST_REPO)
}
stage('Initial env check') {
sh "mkdir -p ${artifacts_dir}"
tempest_result = validate.runCVPtempest(saltMaster, TEMPEST_TARGET_NODE, TEMPEST_TEST_PATTERN, SKIP_LIST_PATH, remote_artifacts_dir, "docker_tempest_initial")
validate.openstack_cleanup(saltMaster, TEMPEST_TARGET_NODE)
if (tempest_result != "finished") {
currentBuild.result = "FAILURE"
throw new Exception("Tempest tests failed")
}
}
stage('Soft Shutdown') {
if (MANUAL_CONFIRMATION.toBoolean() == true) {
stage('Ask for manual confirmation') {
input message: "Are you sure you want to shutdown current vip node?"
}
}
current_target_node = validate.get_vip_node(saltMaster, TARGET_NODES)
if (current_target_node == null) {
throw new Exception("Cannot current vip node in ${TARGET_NODES} nodes")
}
common.warningMsg("Shutdown current vip node ${current_target_node}")
validate.shutdown_vm_node(saltMaster, current_target_node, 'soft_shutdown')
sleep 15
}
stage('Check during shutdown') {
tempest_result = validate.runCVPtempest(saltMaster, TEMPEST_TARGET_NODE, TEMPEST_TEST_PATTERN, SKIP_LIST_PATH, remote_artifacts_dir, "docker_tempest_during_shutdown")
validate.openstack_cleanup(saltMaster, TEMPEST_TARGET_NODE)
if (tempest_result != "finished") {
currentBuild.result = "FAILURE"
throw new Exception("Tempest tests failed")
}
}
stage('Power on') {
common.infoMsg('Powering on node')
kvm = validate.locate_node_on_kvm(saltMaster, current_target_node)
salt.cmdRun(saltMaster, kvm, "virsh start ${current_target_node}")
common.infoMsg("Checking that node is UP")
status = salt.minionsReachable(saltMaster, 'I@salt:master', current_target_node, null, 10, num_retries)
if (status == null) {
throw new Exception("Node ${current_target_node} cannot start")
}
first_node = current_target_node
current_target_node = null
sleep 30
}
stage('Check after shutdown') {
tempest_result = validate.runCVPtempest(saltMaster, TEMPEST_TARGET_NODE, TEMPEST_TEST_PATTERN, SKIP_LIST_PATH, remote_artifacts_dir, "docker_tempest_after_shutdown")
validate.openstack_cleanup(saltMaster, TEMPEST_TARGET_NODE)
if (tempest_result != "finished") {
currentBuild.result = "FAILURE"
throw new Exception("Tempest tests failed")
}
sleep 15
}
stage('Hard Shutdown') {
if (MANUAL_CONFIRMATION.toBoolean() == true) {
stage('Ask for manual confirmation') {
input message: "Are you sure you want to hard shutdown current vip node?"
}
}
salt.cmdRun(saltMaster, first_node, "service keepalived stop")
current_target_node = validate.get_vip_node(saltMaster, TARGET_NODES)
common.warningMsg("Shutdown current vip node ${current_target_node}")
validate.shutdown_vm_node(saltMaster, current_target_node, 'hard_shutdown')
//TODO:if previous command fails, keeaplived will not be started on first_node
sleep 10
salt.cmdRun(saltMaster, first_node, "service keepalived start")
}
stage('Check during hard shutdown') {
tempest_result = validate.runCVPtempest(saltMaster, TEMPEST_TARGET_NODE, TEMPEST_TEST_PATTERN, SKIP_LIST_PATH, remote_artifacts_dir, "docker_tempest_during_hard_shutdown")
validate.openstack_cleanup(saltMaster, TEMPEST_TARGET_NODE)
if (tempest_result != "finished") {
currentBuild.result = "FAILURE"
throw new Exception("Tempest tests failed")
}
}
stage('Power on') {
common.infoMsg('Powering on node')
kvm = validate.locate_node_on_kvm(saltMaster, current_target_node)
salt.cmdRun(saltMaster, kvm, "virsh start ${current_target_node}")
common.infoMsg("Checking that node is UP")
status = salt.minionsReachable(saltMaster, 'I@salt:master', current_target_node, null, 10, num_retries)
if (status == null) {
throw new Exception("Node ${current_target_node} cannot start")
}
current_target_node = null
sleep 30
}
stage('Check after hard shutdown') {
tempest_result = validate.runCVPtempest(saltMaster, TEMPEST_TARGET_NODE, TEMPEST_TEST_PATTERN, SKIP_LIST_PATH, remote_artifacts_dir, "docker_tempest_after_hard_shutdown")
validate.openstack_cleanup(saltMaster, TEMPEST_TARGET_NODE)
if (tempest_result != "finished") {
currentBuild.result = "FAILURE"
throw new Exception("Tempest tests failed")
}
sleep 5
}
stage('Reboot') {
if (MANUAL_CONFIRMATION.toBoolean() == true) {
stage('Ask for manual confirmation') {
input message: "Are you sure you want to reboot current vip node?"
}
}
current_target_node = validate.get_vip_node(saltMaster, TARGET_NODES)
common.warningMsg("Rebooting current vip node ${current_target_node}")
validate.shutdown_vm_node(saltMaster, current_target_node, 'reboot')
sleep 5
}
stage('Check during reboot') {
tempest_result = validate.runCVPtempest(saltMaster, TEMPEST_TARGET_NODE, TEMPEST_TEST_PATTERN, SKIP_LIST_PATH, remote_artifacts_dir, "docker_tempest_during_reboot")
validate.openstack_cleanup(saltMaster, TEMPEST_TARGET_NODE)
if (tempest_result != "finished") {
currentBuild.result = "FAILURE"
throw new Exception("Tempest tests failed")
}
sleep 30
}
stage('Check after reboot') {
common.warningMsg("Checking that node is UP")
status = salt.minionsReachable(saltMaster, 'I@salt:master', current_target_node, null, 10, num_retries)
if (status == null) {
throw new Exception("Node ${current_target_node} cannot start")
}
current_target_node = null
tempest_result = validate.runCVPtempest(saltMaster, TEMPEST_TARGET_NODE, TEMPEST_TEST_PATTERN, SKIP_LIST_PATH, remote_artifacts_dir, "docker_tempest_after")
validate.openstack_cleanup(saltMaster, TEMPEST_TARGET_NODE)
if (tempest_result != "finished") {
currentBuild.result = "FAILURE"
throw new Exception("Tempest tests failed")
}
}
stage('Collect results') {
validate.addFiles(saltMaster, TEMPEST_TARGET_NODE, remote_artifacts_dir, artifacts_dir)
archiveArtifacts artifacts: "${artifacts_dir}/*"
if (DEBUG_MODE == 'false') {
validate.runCleanup(saltMaster, TEMPEST_TARGET_NODE)
salt.cmdRun(saltMaster, TEMPEST_TARGET_NODE, "rm -rf ${remote_artifacts_dir}")
}
}
} finally {
if (DEBUG_MODE == 'false') {
salt.cmdRun(saltMaster, TEMPEST_TARGET_NODE, "rm -rf ${remote_artifacts_dir}")
validate.runCleanup(saltMaster, TEMPEST_TARGET_NODE)
if (current_target_node != null) {
common.warningMsg("Powering on node ${current_target_node}")
kvm = validate.locate_node_on_kvm(saltMaster, current_target_node)
salt.cmdRun(saltMaster, kvm, "virsh start ${current_target_node}")
}
}
}
}
}