-
Notifications
You must be signed in to change notification settings - Fork 2
/
parallel_pipeline.yml
434 lines (412 loc) · 20.5 KB
/
parallel_pipeline.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
def templateNameofET = 'errata-rails-aio-template'
def templateNameofMysql = 'errata-mysql-template'
//def templatePathofET = 'https://gitlab.cee.redhat.com/wlin/errata-upshift/raw/master/upshift/template/errata_services_template/errata-all-services-in-one-pod.yaml'
// def templatePathofMysql = 'https://gitlab.cee.redhat.com/wlin/errata-upshift/raw/master/upshift/template/mysql/mysql_template.yaml'
def templatePathofET = 'https://github.com/testcara/pipeline_file/raw/master/app_template.json'
def templatePathofMysql = 'https://github.com/testcara/pipeline_file/raw/master/mysql_template.json'
def RUN_USER = '1058980001'
def etTemplateParameters = "-p=CPU_LIMITS=1 -p=MEM_REQUEST=4Gi -p=MEM_LIMITS=4Gi -p=ET_CONFIGMAP=et-qe-testing-settings -p=ET_SECRET=et-qe-testing-settings -p=ET_MYSQL_SECRET=et-qe-testing-mysql -p=RUN_USER=$RUN_USER"
def process_template_and_create_objects(String templateName, String templateParameters, String[] objectsName) {
echo "testing---"
echo "$templateName"
def templateGeneratedSelector = openshift.selector(objectsName)
def objectModels = openshift.process(templateName, templateParameters)
def objects
def verb
def objectsGeneratedFromTemplate = templateGeneratedSelector.exists()
if (!objectsGeneratedFromTemplate) {
verb = "Created"
objects = openshift.create(objectModels)
} else {
verb = "Found"
objects = templateGeneratedSelector
}
objects.withEach {
echo "${verb} ${it.name()} from template with labels ${it.object().metadata.labels}"
}
return objects
}
def check_builds(Integer time, String bcName) {
def goodBuildStatus = (String[]) ['Running', 'Pending', 'Complete']
def bcSelector = openshift.selector("bc", bcName)
bcSelector.startBuild()
def builds= bcSelector.related('builds')
timeout(time) {
// Checking watch output and running watch closure again in 250ms
builds.untilEach(1) {
def status = it.object().status.phase
if ( goodBuildStatus.contains(status) == false ) {
throw new Exception("Build failed")
} //if
if ( status == 'Complete' )
{
echo "---> Build Complete ..."
return true
} //if
} //each
} //timeout
}
def check_deployments(Integer time, String dcName) {
def goodBuildStatus = (String[]) ['Running', 'Pending', 'Complete']
def dcSelector = openshift.selector("dc", dcName)
dcSelector.rollout().latest()
timeout(time) {
openshift.selector("dc", dcName).related('pods').untilEach(1) {
if (it.object().status.phase == "Running" )
{
echo "---> Deploy Complete ..."
return true
}
} //each
} // timout
}
def all_stages(String templateNameofMysql, String templateNameofET, String etTemplateParameters, String appName){
stage('create Mysql app') {
container('qe-testing-runner') {
script {
openshift.withCluster('https://paas.psi.redhat.com', 'rA1EA9Cv_jTLE6wtYQBI60OQ10RHOgE5A0eT125vSic') {
openshift.withProject('errata-qe-test'){
echo '--- Create Mysql app ${appName}-mysql from the ET Mysql template --->'
def MYSQL_APP_NAME = "${appName}-mysql"
def mysqlTemplateParameters = "-p=APP_NAME=${MYSQL_APP_NAME}"
def mysqlObjects = (String[]) ["is/${MYSQL_APP_NAME}", "bc/${MYSQL_APP_NAME}", "dc/${MYSQL_APP_NAME}", "svc/${MYSQL_APP_NAME}"]
process_template_and_create_objects(templateNameofMysql, mysqlTemplateParameters, mysqlObjects)
} //project
} //cluster
} //script
} //container
} // stage
stage('build Mysql app') {
container('qe-testing-runner') {
script {
openshift.withCluster('https://paas.psi.redhat.com', 'rA1EA9Cv_jTLE6wtYQBI60OQ10RHOgE5A0eT125vSic') {
openshift.withProject('errata-qe-test'){
echo '--- Build Mysql app: ${appName}-mysql --->'
check_builds(20, "${appName}-mysql")
} //project
} //cluster
} //script
} //container
} // stage
stage('deploy Mysql app') {
container('qe-testing-runner') {
script {
openshift.withCluster('https://paas.psi.redhat.com', 'rA1EA9Cv_jTLE6wtYQBI60OQ10RHOgE5A0eT125vSic') {
openshift.withProject('errata-qe-test'){
echo '--- Deploy Mysql app: ${appName}-mysql --->'
check_deployments(2, "${appName}-mysql")
} //project
} //cluster
} //script
} //container
} // stage
stage('create ET app') {
container('qe-testing-runner') {
script {
openshift.withCluster('https://paas.psi.redhat.com', 'rA1EA9Cv_jTLE6wtYQBI60OQ10RHOgE5A0eT125vSic') {
openshift.withProject('errata-qe-test'){
echo '--- Create ET apps from the ET template --->'
etTemplateParameters = "-p=APP_NAME=${appName} ${etTemplateParameters}"
def etObjects = (String[]) ["is/${appName}-s2i", "is/${appName}-basic", "bc/${appName}-bc", "dc/${appName}-rails", "route/${appName}-route", "svc/${appName}-svc"]
process_template_and_create_objects(templateNameofET, etTemplateParameters, etObjects)
} //project
} //cluster
} //script
} //container
} //stage
stage('build ET app') {
container('qe-testing-runner') {
script {
openshift.withCluster('https://paas.psi.redhat.com', 'rA1EA9Cv_jTLE6wtYQBI60OQ10RHOgE5A0eT125vSic') {
openshift.withProject('errata-qe-test'){
echo '--- Build ET app --->'
check_builds(20, "${appName}-bc")
} //project
} //cluster
} //script
} //container
} // stage
stage('deploy ET app') {
container('qe-testing-runner') {
script {
openshift.withCluster('https://paas.psi.redhat.com', 'rA1EA9Cv_jTLE6wtYQBI60OQ10RHOgE5A0eT125vSic') {
openshift.withProject('errata-qe-test'){
echo '--- Deploy ET app --->'
check_deployments(2, "${appName}-rails")
} //project
} //cluster
} //script
}// container
} //stage
stage('load db and do TS2 test'){
container("qe-testing-runner") {
script{
openshift.withCluster('https://paas.psi.redhat.com', 'rA1EA9Cv_jTLE6wtYQBI60OQ10RHOgE5A0eT125vSic') {
openshift.withProject('errata-qe-test'){
sh '''
import_db(){
oc exec ${1} -i -- mysql -uroot -parNdk123_ -Derrata < /tmp/TS2_db/errata.latest.sql
}
rake_migration(){
oc exec ${1} -i -- bundle exec rake db:migrate
}
update_setting_to_disable_qpid() {
config_file="/opt/app-root/src/config/initializers/settings.rb"
oc exec ${1} -- sed -i "s/:messages_to_qpid_enabled => true/:messages_to_qpid_enabled => false/g" ${config_file}
}
restart_services() {
oc exec ${1} -- /etc/init.d/httpd24-httpd restart
oc exec ${1} -- /etc/init.d/delayed_job restart
oc exec ${1} -- /etc/init.d/messaging_service restart
}
reset_testing_host(){
sed -i "s/et-system-test-qe-01.usersys.redhat.com/${1}.cloud.paas.psi.redhat.com/g" ${2}/features/remote/config/env.yml
}
specify_et_umb_for_cucumber_umb_cases(){
umb_config_1='/opt/app-root/src/config/initializers/credentials/message_bus.rb'
umb_config_2='/opt/app-root/src/examples/ruby/message_bus/umb_configuration.rb'
et_pod=$(sed -n '1 p' et_pod_names)
for umb_config in $umb_config_1 $umb_config_2
do
oc exec ${et_pod} -i -- sed -i "s/ENV\\['ET_UMB_BROKER_URL_1'\\]/'amqp:\\/\\/cucumber-umb-qe:5672'/g" ${umb_config}
oc exec ${et_pod} -i -- sed -i "s/ENV\\['ET_UMB_BROKER_URL_2'\\]/'amqp:\\/\\/cucumber-umb-qe:5672'/g" ${umb_config}
done
restart_services ${et_pod}
}
specify_runner_umb_for_cucumber_umb_cases(){
umb_config_1='TS2_testing_group_1/features/remote/support/umb.rb'
sed -i "s/umb-qe/cucumber-umb-qe/g" $umb_config_1
}
collect_pods_info(){
if [ ! -f et_pod_names ]
then
oc get pods | grep "cucumber" | grep "rails" | grep Running | grep -v build | grep -v deploy | awk '{print $1}'| tail -n3 > et_pod_names
fi
if [ ! -f mysql_pod_names ]
then
oc get pods | grep "cucumber" | grep "mysql" | grep Running | grep -v build | grep -v deploy | awk '{print $1}'| tail -n3 > mysql_pod_names
fi
}
write_tags(){
if [ ! -f tags_name ]
then
echo -e "-t @umb,@examples\n-t @mx,@module\n-t ~@disable -t ~@real_e2e --tags ~@pdc -t ~@mx -t ~@module -t ~@umb -t ~@examples" > tag_groups
fi
}
prepare_umb_for_first_group(){
if [ ! -f set_umb_1 ]
then
touch set_umb_1
specify_et_umb_for_cucumber_umb_cases
specify_runner_umb_for_cucumber_umb_cases
fi
}
run_import_sql_to_db(){
echo "---> Import database ..."
cat mysql_pod_names | while read mysql_pod
do
if [ ! -f $mysql_pod ]
then
touch $mysql_pod
import_db $mysql_pod
break
fi
done
}
run_migration_and_disable_sending_qpid_msg(){
echo "---> DB migraion and disable to send qpid message ..."
cat et_pod_names | while read et_pod
do
if [ ! -f $et_pod ]
then
touch $et_pod
rake_migration $et_pod
update_setting_to_disable_qpid $et_pod
restart_services $et_pod
break
fi
done
}
write_test_scripts_and_do_testing(){
for group_num in 1 2 3
do
if [ ! -f cucumber_${group_num}.sh ]
then
cd TS2_testing_group_${group_num}
bundle install --path=/opt/rh/rh-ruby22/root/usr/local/bin
cd -
pod=$(sed -n "${group_num} p" et_pod_names)
app_name=$(echo $pod | cut -d '-' -f 1-3)
tags=$(sed -n "${group_num} p" tag_groups)
reset_testing_host ${app_name} TS2_testing_group_${group_num}
sed -i "s/[email protected]/[email protected]/g" TS2_testing_group_${group_num}/features/remote/config/env.yml
cucumber_command="ET_POD=${pod} RUN_ON_PSI=True TEST_ENV=qe_01 BZ_ADMIN_PASSWD=1HSSQE@redhat bundle exec cucumber -p remote"
echo ${cucumber_command} ${tags} --format json_pretty --strict -o cucumber-report-${group_num}.json TS2_testing_group_${group_num}/features/remote > cucumber_${group_num}.sh
chmod +x cucumber_${group_num}.sh
cucumber_${group_num}.sh
break
fi
done
}
if [ ${QE_Testing} == 'true' ]
then
cwd=$(pwd)
collect_pods_info
write_tags
run_import_sql_to_db
run_migration_and_disable_sending_qpid_msg
cd $cwd
prepare_umb_for_first_group
write_test_scripts_and_do_testing
fi
'''
} //project
} //cluster
} //script
} //container
} //stage
} //stages
def prepare_runner() {
stage('prepare runner with gems'){
container("qe-testing-runner") {
script{
openshift.withCluster('https://paas.psi.redhat.com', 'rA1EA9Cv_jTLE6wtYQBI60OQ10RHOgE5A0eT125vSic') {
openshift.withProject('errata-qe-test'){
sh '''
prepare_dir(){
mkdir ${1}
cd ${1} && clone_files
cd -
}
clone_files(){
wget https://code.engineering.redhat.com/gerrit/changes/170351/revisions/8aac6cca887c01b77ad530b7d40b2f321939b15c/archive?format=tgz --no-check-certificate
tar -xvf 'archive?format=tgz'
}
echo "---> Preparing source code for cucumber cases of different groups..."
for dir_num in {1..4}
do
prepare_dir TS2_testing_group_${dir_num}
done
echo "---> Install the gems for cucumber cases ..."
cd TS2_testing_group_4
export RAILS_ENV=test
bundle install --path=/opt/rh/rh-ruby22/root/usr/local/bin
'''
} //project
} //cluster
} //script
} //container
} //stage
} //prepare_runner()
pipeline {
agent { label 'qe-testing-runners'}
parameters {
string(name: 'QE_Testing', defaultValue: 'true', description: 'Do QE Testing')
}
stages {
stage('preamble') {
steps {
container('qe-testing-runner') {
script {
openshift.withCluster('https://paas.psi.redhat.com', 'rA1EA9Cv_jTLE6wtYQBI60OQ10RHOgE5A0eT125vSic') {
openshift.withProject('errata-qe-test'){
echo "Using project: ${openshift.project()}"
sh '''
oc version
'''
} //project
} //cluster
} //script
} //container
} //steps
} //stage
stage('cleanup') {
steps {
container('qe-testing-runner') {
script {
openshift.withCluster('https://paas.psi.redhat.com', 'rA1EA9Cv_jTLE6wtYQBI60OQ10RHOgE5A0eT125vSic') {
openshift.withProject('errata-qe-test'){
echo "--- Delete apps --->"
def etAppNames = ["cucumber-et-1", "cucumber-et-2", "cucumber-et-3"]
def mysqlAppNames = ["cucumber-et-1-mysql", "cucumber-et-2-mysql", "cucumber-et-3-mysql"]
etAppNames.each { app ->
openshift.selector("all", [ app : "$app" ]).delete()
}
mysqlAppNames.each { app ->
openshift.selector("all", [ app : "$app" ]).delete()
}
def exist1 = openshift.selector("template", "$templateNameofET").exists()
if (exist1) {
echo "--- Delete ET template --->"
openshift.selector("template", "$templateNameofET").delete()
} //if
def exist2 = openshift.selector("template", "$templateNameofMysql").exists()
if (exist2) {
echo "--- Delete Mysql template --->"
openshift.selector("template", "$templateNameofMysql").delete()
}
} //project
} //cluster
} //script
} //container
} //steps
} //stage
stage('create templates') {
steps {
container('qe-testing-runner') {
script {
openshift.withCluster('https://paas.psi.redhat.com', 'rA1EA9Cv_jTLE6wtYQBI60OQ10RHOgE5A0eT125vSic') {
openshift.withProject('errata-qe-test'){
echo '--- Upload ET template --->'
openshift.create(templatePathofET)
echo '--- Upload Mysql template --->'
openshift.create(templatePathofMysql)
} //project
} //cluster
} //script
} //container
} //steps
} // stage
stage('create apps and do TS2 testing'){
parallel {
stage('create first group app and do testing') {
steps {
script{
all_stages(templateNameofMysql, templatePathofET, etTemplateParameters, "cucumber-et-1")
}
}
}
stage('create second group app and do testing') {
steps {
script{
echo "second"
all_stages(templateNameofMysql, templatePathofET, etTemplateParameters, "cucumber-et-2")
}
}
}
stage('create third group app and do testing'){
steps {
script{
echo "third"
all_stages(templateNameofMysql, templatePathofET, etTemplateParameters, "cucumber-et-3",)
}
}
}
stage('prepare the testing runner for cucumber cases'){
steps {
script{
echo "fourth"
prepare_runner()
}
}
}
} //parallel
}// stage
} //stages
post {
always {
archiveArtifacts '**/cucumber-report.json'
cucumber fileIncludePattern: "**/cucumber-report*.json", sortingMethod: "ALPHABETICAL"
}
}
}