-
Notifications
You must be signed in to change notification settings - Fork 301
609 lines (559 loc) · 29.7 KB
/
smoketest.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
# Smoketest github action
# =======================
#
# Uses cached instances of previous builds to perform partial (and therefore quicker) incremental builds.
# Once the platform is built and installed (make install),
# it runs the regression suite setup stage within the same github job ('build-and-setup').
#
# On success, an artifact is built, that contains the install binaries and
# the HPCC binaries that have accumulated from the setup stage.
#
# Once the 'build-and-setup' job is complete, the dependent regression suite jobs and unittest job are launched in parallel.
# The regression suite queries are manually sub-divided into chunks (alphabeticaly), and run via a job matrix for parallism.
# If anything fails, all jobs are aborted (fail-fast=true), and the logs are captured into a published artifact.
#
# NOTES:
# + pre-requisite build dependencies (and runtime dependencies) are listed manually, and must be kept up to date with the
# requirements of the platform. MORE: a list of required build dependencies could be kept within the platform source and picked up
# + 'cacheversion' is purely in case it is necessary to force a cache-miss, i.e. all cached items are based on this version
# + Caching is via github's actions/cache, and is limited to 5GB total per repository, with oldest ejected first, and/or >7 days
# cached builds are tagged with:
# 1) base+ref+SHA (exact match, e.g. hpccbuild-1-7.12.10-6c981c48ae6e35b62d86d8e59e42799c5cd14812)
# 2) base_ref (branch match, e.g. hpccbuild-1-7.12.10)
# 3) base-ref major-minor only (e.g. hpccbuild-1-7.12.)
# 4) base-ref major only (e.g. hpccbuild-1-7.)
# 5) generic cacheversion only (e.g. hpccbuild-1-)
# The cache will initially try to match an exact key match (only true if rerunning PR and SHA same),
# and will then failover to trying to find a match of 2-5, in that order.
name: smoketest
env:
cacheversion: 3
VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,read"
on:
pull_request:
branches:
- "master"
- "candidate-*"
- "!candidate-7.6.*"
- "!candidate-7.4.*"
- "!candidate-7.2.*"
- "!candidate-7.0.*"
- "!candidate-6.*"
# NB: this fails to cancel in-flight actions, with error 'Resource not accessible by integration',
# but it's non-fatal. Appears to be dependent on what permissions the PR owner has, i.e. the github token used
# does not have permission to cancel actions.
jobs:
check-skip:
# continue-on-error: true # Uncomment once integration is finished
runs-on: ubuntu-20.04
# Map a step output to a job output
outputs:
platform: ${{ contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') || steps.skip_check.outputs.platform }}
eclwatch: ${{ contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') || steps.skip_check.outputs.eclwatch }}
steps:
- id: skip_check
uses: hpcc-systems/github-actions/changed-modules@main
with:
github_token: ${{ github.token }}
build-and-setup:
name: Build platform and regress setup
needs: check-skip
if: ${{ needs.check-skip.outputs.platform || needs.check-skip.outputs.eclwatch }}
runs-on: ubuntu-20.04
timeout-minutes: 150 # the build step has its own timeout, however the job can sometimes take time to download dependencies
outputs:
cache-state: ${{ steps.check-cache.outputs.state }}
steps:
- name: Setup and git clone
id: vars
run: |
if [[ "${{ github.event_name }}" = 'pull_request' ]]
then
echo ::set-output name=branch::${{ github.head_ref }}
echo ::set-output name=base_ref::${{ github.base_ref }}
versioned=$(echo ${{ github.base_ref }} | sed -E -n 's/^(.*)-([0-9]+)\.([0-9]+)\.([0-9]+|x)(-[0-9]+|).*$/\1 \2 \3 \4 \5/p')
if [[ "${versioned}" ]]
then
echo "matched version = ${versioned}"
IFS=' ' read prefix major minor point cand <<< ${versioned}
echo "name=base-majorminor-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}.${minor}."
echo "name=base-majoronly-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}."
echo ::set-output name=base-majorminor-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}.${minor}.
echo ::set-output name=base-majoronly-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}.
fi
else
echo ::set-output name=branch::${GITHUB_REF#refs/heads/}
echo ::set-output name=base_ref::${GITHUB_REF#refs/heads/}
fi
if [[ "${{ github.event.pull_request.head.repo.owner.login }}" = "${{ github.repository_owner }}" ]]
then
repository=${{ github.repository }}
else
repository=$(echo -n ${{ github.event.pull_request.head.repo.owner.login }}/ ; echo -n ${{ github.repository }} | sed -E -n 's@^[^/]+/(.+)$@\1@p')
fi
echo ::set-output name=repository::${repository}
echo "Cloning https://github.com/${repository}"
git clone -n https://github.com/${repository} src
if [[ "${{ github.event_name }}" = 'pull_request' ]]
then
cd src
git remote add upstream https://github.com/${{ github.repository }}
git fetch upstream +${{ github.ref }}
fetch_sha=$(git rev-parse --verify FETCH_HEAD)
git fetch upstream ${{ github.base_ref}}
base_ref_sha=$(git rev-parse upstream/${{ github.base_ref }})
echo ::set-output name=base_ref_sha::${base_ref_sha}
echo ::set-output name=sha::${fetch_sha}
echo "base_ref_sha = ${base_ref_sha}"
echo "fetch_sha = ${fetch_sha}"
else
echo ::set-output name=sha::${{ github.sha }}
echo ::set-output name=base_ref_sha::${{ github.sha }}
fi
# NB: restore-keys set to e.g. hpccbuild-2-7.12.10, hpccbuild-2-7.12, hpccbuild-2-7, hpccbuild-2-
- name: Fetch cache
id: cache
uses: actions/cache@v2
with:
path: |
build-cache
merge-patch
src-cache
key: hpccbuild-${{ env.cacheversion }}-${{ steps.vars.outputs.base_ref }}-${{ steps.vars.outputs.sha }}
restore-keys: |
hpccbuild-${{ env.cacheversion }}-${{ steps.vars.outputs.base_ref }}
${{ steps.vars.outputs.base-majorminor-restore-key }}
${{ steps.vars.outputs.base-majoronly-restore-key }}
hpccbuild-${{ env.cacheversion }}-
- name: tracing
run: |
echo "Branch = ${{ steps.vars.outputs.branch }}"
echo "Base ref = ${{ steps.vars.outputs.base_ref }}"
echo "SHA = ${{ steps.vars.outputs.sha}}"
echo "Action = ${{ github.action }}"
echo "Event = ${{ github.event_name }}"
echo "Actor = ${{ github.actor }}"
echo "Ref = ${{ github.ref }}"
echo "Sha = ${{ github.sha }}"
echo "github.repository = ${{ github.repository }}"
echo "repository = ${{ steps.vars.outputs.repository }}"
echo "repository_owner = ${{ github.repository_owner }}"
echo "github.workspace = ${{ github.workspace }}"
echo "runner.workspace = ${{ runner.workspace }}"
echo "github.event.pull_request.head.repo.owner.login = ${{ github.event.pull_request.head.repo.owner.login }}"
if [[ "${{ github.event_name }}" = 'pull_request' ]]
then
echo "PR base_ref = ${{ github.base_ref }}"
echo "PR head_ref = ${{ github.head_ref }}"
echo "PR base SHA = ${{ github.event.pull_request.base.sha }}"
fi
echo "restore-key1: hpccbuild-${{ env.cacheversion }}-${{ steps.vars.outputs.base_ref }}"
echo "restore-key2: ${{ steps.vars.outputs.base-majorminor-restore-key }}"
echo "restore-key3: ${{ steps.vars.outputs.base-majoronly-restore-key }}"
echo "restore-key4: hpccbuild-${{ env.cacheversion }}-"
# NB: actions/cache does not set cache-hit to true if restore-keys used.
# So here we:
# 1) check if a cache was restored
# 2) check that the SHA stored in the cache can be found in git
# 3) mv cache into place (build-cache->build)
# 4) checkout source @ base_ref
# 5) get submodules
# 6) apply git merge-patch from cache, bringing source up-to-date with the source used in the cached binary build
# 7) copy src-cache files into place
# 8) touch all files so that 'old' [before original make timestamp], i.e. so behind objects in cache
# 9) checkout pr branch - ensuring only pr files that differ are updated (and hence will rebuild)
# 10) list updated files (for tracing only)
# 11) set 'state' output variable to 'miss' if not absolute match,
# and set to 'hit' if cache is usable. i.e. if set at all, run build and tests.
- name: Check cache
id: check-cache
if: ${{ (steps.cache.outputs.cache-hit != 'true') || contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') }}
run: |
if [[ ("${{ steps.cache.outputs.cache-hit }}" == 'true') && ("${{ contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') }}" == 'true') ]]
then
echo "Exact cache match, but 'smoketest-force-rerun' label forced rerun"
fi
echo ::set-output name=state::miss
if [[ "${{ contains(github.event.pull_request.labels.*.name, 'smoketest-ignore-cache') }}" == 'true' ]]
then
echo "Cache found, but ignored because 'smoketest-ignore-cache' label set"
rm -rf build-cache merge-patch src-cache
else
if [[ -d build-cache ]]
then
echo "Attempting to use existing cache"
stat build-cache merge-patch src-cache
du -sh build-cache merge-patch src-cache
cd src
if [[ "commit" == "$(git cat-file -t $(cat ../build-cache/cache-sha))" ]]
then
mv ../build-cache ../build
echo "Checking out cache SHA (cached github.base_ref SHA): $(cat ../build/cache-sha)"
git checkout $(cat ../build/cache-sha)
git submodule update --init --recursive --jobs 4
echo "Applying merge patch"
git apply ../merge-patch
# Add all mods that came from merge-patch into a commit,
# so that the final git checkout will not clash with working tree files
git add -A
git -c user.name='Smoketest' -c user.email='[email protected]' commit -m 'merge-patch'
cd ../src-cache
echo "Cached source tree files:"
find . -type f
echo "================"
find . -type f | cpio -p -dum ../src
cd ../src
find . -type f -exec touch -r ../build/cache-timestamp {} +
git checkout ${{ steps.vars.outputs.sha }}
git submodule update --init --recursive --jobs 4
echo "Changed files (from SHA: $(cat ../build/cache-sha)):"
find -name '.git*' -prune -o -newer ../build/cache-timestamp -type f -print
echo ::set-output name=state::hit
else
echo "Cached SHA $(cat ../build-cache/cache-sha) could not be found). Skipping cache."
fi
fi
fi
- name: Dependencies20.04
if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set then step needed
run: |
sudo apt-get update
sudo apt-get -yq install bison flex build-essential binutils-dev libldap2-dev libcppunit-dev libicu-dev libxslt1-dev \
zlib1g-dev libboost-regex-dev libarchive-dev libv8-dev default-jdk libapr1-dev libaprutil1-dev libiberty-dev \
libhiredis-dev libtbb-dev libxalan-c-dev libnuma-dev libevent-dev libatlas-base-dev libblas-dev python3-dev \
default-libmysqlclient-dev libsqlite3-dev libmemcached-dev libcurl4-openssl-dev pkg-config libtool autotools-dev automake \
libssl-dev xmlstarlet
curl https://deb.nodesource.com/gpgkey/nodesource.gpg.key | sudo apt-key add -
sudo apt-add-repository "deb https://deb.nodesource.com/node_16.x $(lsb_release -sc) main"
sudo apt-get update -y
sudo apt-get install -y nodejs
echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)"
- name: Cache miss prep
if: ${{ steps.check-cache.outputs.state == 'miss' }}
run: |
rm -rf build build-cache merge-patch src-cache
mkdir build
mkdir src-cache
cd src
git checkout ${{ steps.vars.outputs.sha }}
git submodule update --init --recursive --jobs 4
- name: "Setup NuGet credentials"
if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set then step needed
run: |
mono `vcpkg fetch nuget | tail -n 1` \
sources add \
-source "https://nuget.pkg.github.com/hpcc-systems/index.json" \
-storepasswordincleartext \
-name "GitHub" \
-username "${{ github.actor }}" \
-password "${{ secrets.GITHUB_TOKEN }}"
- name: "Remove builtin vcpkg"
if: ${{ github.event_name == matrix.event_name && needs.check-skip.outputs.platform }}
shell: "bash"
run: |
sudo rm -rf "$VCPKG_INSTALLATION_ROOT"
- name: "vcpkg Bootstrap"
if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set then step needed
working-directory: ./src/vcpkg
run: |
./bootstrap-vcpkg.sh
# Subsequent cache hit builds use cache-timestamp to ensure all sources are marked 'old',
# except those changed between the cache SHA and the SHA being built.
# NB: BUILD_TAG is overridden to a constant "smoketest", this is to prevent the auto-generated build tag being
# used, which would cause it to change per PR (per SHA), and because it's in a common header, cause a lot of undesirable rebuilding.
- name: Build
timeout-minutes: 120 # Generous, typical build time from clean is < 60
if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set to anything, then step needed
run: |
cd build
echo "removing CMakeCache.txt"
rm -f CMakeCache.txt
if [[ ! -f cache-timestamp ]]; then touch cache-timestamp; fi
touch ../cmake-timestamp
`../src/vcpkg/vcpkg fetch cmake | tail -n 1` ../src -Wno-dev -DRUNTIME_USER=$(whoami) -DRUNTIME_GROUP=$(id -gn) -DDESTDIR=$(realpath ..)/install -DINCLUDE_PLUGINS=1 -DCMAKE_BUILD_TYPE=Release -DUSE_PYTHON2=0 -DUSE_PYTHON3=1 -DSUPPRESS_SPARK=1 -DUSE_CPPUNIT=1 -DUSE_CASSANDRA=Off -DCMAKE_CXX_FLAGS="-DBUILD_TAG=\\\"dummytag\\\" -DBUILD_VERSION_MAJOR=1 -DBUILD_VERSION_MINOR=1 -DBUILD_VERSION_POINT=0" -DBUILD_TAG="smoketest" -DSTRIP_RELEASE_SYMBOLS=1
make -j$(nproc) install
- name: build-logs-artifact
if: ${{ failure() }}
uses: actions/upload-artifact@v2
with:
name: build-logs
path: |
build/CMakeCache.txt
build/CMakeFiles/CMakeOutput.log
build/CMakeFiles/CMakeError.log
# Cache contains:
# - make binaries (build-cache)
# - Any sources written since cache-timestamp within the source dir (src-cache) (e.g. AWS writes a config header)
# - cache-timestamp, timestamp of cache-miss build. All files except diffs set to this time, ensuring no existing binaries are rebuilt
# - cache-sha, the SHA of the base_ref used to build. On a cache hit, used to find changed files and ensure they are only ones rebuilt.
# - merge-path, the diffs between the base_ref and the PR commit at the time of PR/cache build.
- name: Prepare cache
if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set to anything, then step needed
run: |
echo ${{ steps.vars.outputs.base_ref_sha }} > build/cache-sha
mv build build-cache
cd src
find -name '.git*' -prune -o -newer ../cmake-timestamp -type f -print | cpio -p -dum ../src-cache
echo "Merge patch files:"
git diff --name-only ${{ steps.vars.outputs.base_ref_sha }} ${{ steps.vars.outputs.sha }}
git diff --binary ${{ steps.vars.outputs.base_ref_sha }} ${{ steps.vars.outputs.sha }} > ../merge-patch
# - alter stock ecl-test.json, to point to install dir (using jq)
# - create a clean copy of the install directory (for later reuse)
# - rm hpcc-data from copy (the post-setup hpcc-data will be moved in when done)
- name: Prepare install artifact
if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }}
run: |
# configure ecl-test.json with correct paths
jq --arg rootdir "${{ github.workspace }}" \
'.Regress.dropzonePath = $rootdir + "/install" + .Regress.dropzonePath | .Regress.setupExtraParams.OriginalTextFilesOsPath = $rootdir + "/install" + .Regress.setupExtraParams.OriginalTextFilesOsPath | .Regress.setupExtraDParams.HPCCBaseDir = $rootdir + "/install" + .Regress.setupExtraDParams.HPCCBaseDir | .Regress.regressionDir = $rootdir + "/regress" | .Regress.maxAttemptCount = "1" | .Regress.logDir = $rootdir + "/regress/log"' \
install/opt/HPCCSystems/testing/regress/ecl-test.json > ecl-test.json
mv -f ecl-test.json install/opt/HPCCSystems/testing/regress/ecl-test.json
# configure environment.xml to slavesPerNode=2, channelsPerNode=1
xmlstarlet ed -L -u 'Environment/Software/ThorCluster/@slavesPerNode' -v 2 -u 'Environment/Software/ThorCluster/@channelsPerSlave' -v 1 install/etc/HPCCSystems/environment.xml
cp ${{ github.workspace }}/src/.github/workflows/timeoutcmd install/opt/HPCCSystems/bin/
cp ${{ github.workspace }}/src/.github/workflows/smoketest-preabort.sh install/opt/HPCCSystems/bin/
mkdir copy
cp -rp install copy/
rm -rf copy/install/var/lib/HPCCSystems/hpcc-data
rm -rf copy/install/var/lib/HPCCSystems/queries
- name: Run regression suite setup
if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }}
timeout-minutes: 10
run: |
export LANG="en_US.UTF-8"
sudo update-locale
source install/opt/HPCCSystems/sbin/hpcc_setenv
install/opt/HPCCSystems/etc/init.d/hpcc-init start
mkdir -p regress
cd install/opt/HPCCSystems/testing/regress
# force regression suite to timeout after 8 minutes, so it captures ZAP report of any inflight hung queries
timeoutcmd $((8 * 60)) \
./ecl-test setup --preAbort 'smoketest-preabort.sh' --pq 2 --generateStackTrace
grep Failure: ${{ github.workspace }}/regress/log/setup_*.log
if [[ "$(grep -ohP '(?<=^Failure: )[0-9]+$' ${{ github.workspace }}/regress/log/setup_*.log | awk '{s+=$1} END {print s}')" -gt 0 ]]; then exit 1; fi
${{ github.workspace }}/install/opt/HPCCSystems/etc/init.d/hpcc-init stop
- name: regression-setup-logs-artifact
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v2
with:
name: regress-setup-logs
path: |
install/var/log/HPCCSystems
regress/
# - mv regression suite setup created data from hpcc-data and queries into copy of install
# - create tarball of install for artifact uploading
- name: Finalize install artifact
if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }}
run: |
mv install/var/lib/HPCCSystems/hpcc-data copy/install/var/lib/HPCCSystems/
mv install/var/lib/HPCCSystems/queries copy/install/var/lib/HPCCSystems/
[[ -d ${{ github.workspace }}/src/esp/src/test-ui ]] && cp -r ${{ github.workspace }}/src/esp/src/test-ui copy/install/
cd copy
tar --zstd -cf ../install.tgz install
- name: Upload install artifact
if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }}
uses: actions/upload-artifact@v2
with:
name: installsetup-artifact
path: |
install.tgz
# Matrix of jobs run in parallel once build+setup above completes successfully.
# All use the post-setup install.tgz artifact, that contains binaries and setup state
# Break each engine run into sets for speed
# NB: each regression suite job, runs these steps:
# - installs dependencies (probably too many for runtime)
# - Starts HPCC
# - Runs regression suite with params { engine, match[pattern,exclude] }
# TODO: needs to process results, capture report, to be assembled by workflow when all jobs done
regression-jobs:
needs: build-and-setup
if: ${{ needs.build-and-setup.outputs.cache-state != '' && github.event_name == 'pull_request' && needs.check-skip.outputs.platform }}
timeout-minutes: 60 # each matrix step has its own timeout, however the job can sometimes take time to download dependencies
runs-on: ubuntu-20.04
strategy:
fail-fast: true
matrix:
engine: ['hthor', 'thor', 'roxie']
match:
- pattern: '[a-d]*.ecl'
- pattern: '[e-h]*.ecl'
- pattern: '[i-l]*.ecl'
- pattern: '[m-q]*.ecl'
- pattern: '[r-u]*.ecl'
exclude: teststdlibrary.ecl
- pattern: '[v-z]*.ecl'
include:
- engine: 'hthor'
match:
pattern: teststdlibrary.ecl
- engine: 'thor'
match:
pattern: teststdlibrary.ecl
- engine: 'roxie'
match:
pattern: teststdlibrary.ecl
steps:
- uses: actions/download-artifact@v2
with:
name: installsetup-artifact
- name: vars
id: vars
run: |
echo ::set-output name=matrix-setname::$(echo -n ${{ matrix.match.pattern }} | tr -c "[:alnum:]" _)
- name: Prerequisites
run: |
sudo apt-get update
sudo apt-get -yq install bison flex build-essential binutils-dev libldap2-dev libcppunit-dev libicu-dev libxslt1-dev \
zlib1g-dev libboost-regex-dev libarchive-dev libv8-dev default-jdk libapr1-dev libaprutil1-dev libiberty-dev \
libhiredis-dev libtbb-dev libxalan-c-dev libnuma-dev libevent-dev libatlas-base-dev libblas-dev python3-dev \
default-libmysqlclient-dev libsqlite3-dev libmemcached-dev libcurl4-openssl-dev pkg-config libtool autotools-dev automake \
libssl-dev gdb
curl https://deb.nodesource.com/gpgkey/nodesource.gpg.key | sudo apt-key add -
sudo apt-add-repository "deb https://deb.nodesource.com/node_16.x $(lsb_release -sc) main"
sudo apt-get update -y
sudo apt-get install -y nodejs
echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)"
- name: run
timeout-minutes: 30 # generous, each matrix step shouldn't take more than 10 mins
run: |
export LANG="en_US.UTF-8"
sudo update-locale
tar --zstd -xf install.tgz
source install/opt/HPCCSystems/sbin/hpcc_setenv
install/opt/HPCCSystems/etc/init.d/hpcc-init start
cd install/opt/HPCCSystems/testing/regress
(cd ecl; ls ${{ matrix.match.pattern }}) > matches.tmp
echo queries are:
cat matches.tmp
if [[ -z "${{ matrix.match.exclude }}" ]]
then
queries="$(cat matches.tmp | tr '\n' ' ')"
else
queries="$(cd ecl; ls ${{ matrix.match.exclude }} | grep -v -f - ../matches.tmp | tr '\n' ' ')"
fi
echo queries after exclude: ${queries}
# force regression suite to timeout after 28 minutes, so it captures ZAP report of any inflight hung queries
timeoutcmd $((28 * 60)) \
./ecl-test query --preAbort 'smoketest-preabort.sh' --pq 2 --target ${{ matrix.engine }} --excludeclass python2,embedded-r,embedded-js,3rdpartyservice,mongodb --generateStackTrace ${queries}
grep Failure: ${{ github.workspace }}/regress/log/${{ matrix.engine }}.*.log
if [[ "$(grep -oP '(?<=^Failure: )[0-9]+$' ${{ github.workspace }}/regress/log/${{ matrix.engine }}.*.log)" -gt 0 ]]; then exit 1; fi
- name: regression-run-logs-artifact
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v2
with:
name: regression-run-logs-${{ matrix.engine }}-${{ steps.vars.outputs.matrix-setname }}
path: |
install/var/log/HPCCSystems
regress/
if-no-files-found: error
# NB: this doesn't really need the post-setup data files included in install.tgz
# but as this is relatively quick and in parallel with others, it probably doesn't matter
unittests:
needs: build-and-setup
if: ${{ needs.build-and-setup.outputs.cache-state != '' && github.event_name == 'pull_request' }}
timeout-minutes: 30 # the unittests run step has its own timeout, however the job can sometimes take time to download dependencies
runs-on: ubuntu-20.04
steps:
- uses: actions/download-artifact@v2
with:
name: installsetup-artifact
- name: Prerequisites
run: |
sudo apt-get update
sudo apt-get -yq install bison flex build-essential binutils-dev libldap2-dev libcppunit-dev libicu-dev libxslt1-dev \
zlib1g-dev libboost-regex-dev libarchive-dev libv8-dev default-jdk libapr1-dev libaprutil1-dev libiberty-dev \
libhiredis-dev libtbb-dev libxalan-c-dev libnuma-dev libevent-dev libatlas-base-dev libblas-dev python3-dev \
default-libmysqlclient-dev libsqlite3-dev libmemcached-dev libcurl4-openssl-dev pkg-config libtool autotools-dev automake \
libssl-dev gdb
curl https://deb.nodesource.com/gpgkey/nodesource.gpg.key | sudo apt-key add -
sudo apt-add-repository "deb https://deb.nodesource.com/node_16.x $(lsb_release -sc) main"
sudo apt-get update -y
sudo apt-get install -y nodejs
echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)"
- name: run
timeout-minutes: 10 # generous, expected time is approx 1 min.
run: |
export LANG="en_US.UTF-8"
sudo update-locale
tar --zstd -xf install.tgz
install/opt/HPCCSystems/bin/unittests
# NB: this doesn't really need the post-setup data files included in install.tgz
# but as this is relatively quick and in parallel with others, it probably doesn't matter
eclwatch-ui-tests:
needs: build-and-setup
if: ${{ needs.build-and-setup.outputs.cache-state != '' && github.event_name == 'pull_request' && needs.check-skip.outputs.eclwatch }}
timeout-minutes: 30 # the ui-tests run step has its own timeout, however the job can sometimes take time to download dependencies
runs-on: ubuntu-20.04
steps:
- uses: actions/download-artifact@v2
with:
name: installsetup-artifact
- name: Check-ECLWatch-UI-Test-Directory
id: check
run: |
tar --zstd -xf install.tgz
if [[ ! -d install/test-ui/tests ]]
then
echo "ECLWatch UI test-ui/tests directory missing."
else
javaFilesCount=$(find install/test-ui/tests/ -iname '*.java' -type f -print | wc -l )
echo "Number of test java files is $javaFilesCount"
if [[ ${javaFilesCount} -eq 0 ]]
then
echo "No java files, do nothing."
else
echo ::set-output name=runtests::true
fi
fi
- name: Prerequisites
if: steps.check.outputs.runtests
run: |
sudo apt-get update
sudo apt-get install -y git wget net-tools
sudo apt-get install -y tzdata unzip xvfb libxi6
sudo apt-get install -y default-jdk
sudo apt-get -yq install bison flex build-essential binutils-dev libldap2-dev libcppunit-dev libicu-dev libxslt1-dev \
zlib1g-dev libboost-regex-dev libarchive-dev libv8-dev default-jdk libapr1-dev libaprutil1-dev libiberty-dev \
libhiredis-dev libtbb-dev libxalan-c-dev libnuma-dev libevent-dev libatlas-base-dev libblas-dev python3-dev \
default-libmysqlclient-dev libsqlite3-dev libmemcached-dev libcurl4-openssl-dev pkg-config libtool autotools-dev automake \
libssl-dev gdb
curl https://deb.nodesource.com/gpgkey/nodesource.gpg.key | sudo apt-key add -
sudo apt-add-repository "deb https://deb.nodesource.com/node_16.x $(lsb_release -sc) main"
sudo apt-get update -y
sudo apt-get install -y nodejs
echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)"
- name: UI-Test-Prerequisites
if: steps.check.outputs.runtests
run: |
wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
sudo apt-get install -y ./google-chrome-stable_current_amd64.deb
wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip
unzip chromedriver_linux64.zip
sudo mv chromedriver /usr/bin/chromedriver
sudo chown root:root /usr/bin/chromedriver
sudo chmod +x /usr/bin/chromedriver
wget https://selenium-release.storage.googleapis.com/3.141/selenium-server-standalone-3.141.59.jar
wget http://www.java2s.com/Code/JarDownload/testng/testng-6.8.7.jar.zip
unzip testng-6.8.7.jar.zip
- name: run
timeout-minutes: 10 # generous, current runtime is ~1min, this should be increased if new tests are added
if: steps.check.outputs.runtests
run: |
export LANG="en_US.UTF-8"
sudo update-locale
source install/opt/HPCCSystems/sbin/hpcc_setenv
install/opt/HPCCSystems/etc/init.d/hpcc-init start
export CLASSPATH=".:$(realpath selenium-server-standalone-3.141.59.jar):$(realpath testng-6.8.7.jar)"
pushd install/test-ui
./run.sh tests http://localhost:8010 > eclWatchUiTest.log 2>&1
retCode=$?
echo "UI test done"
[[ $retCode -ne 0 ]] && exit 1
popd
- name: eclwatch-ui-test-logs-artifact
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v2
with:
name: ECLWatchUiTest
path: install/test-ui/eclWatchUiTest.log
if-no-files-found: error