forked from broadinstitute/gatk
-
Notifications
You must be signed in to change notification settings - Fork 0
/
.travis.yml
225 lines (221 loc) · 10.7 KB
/
.travis.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
language: java
dist: bionic
group: travis_lts
git:
# We intentionally check out the entire repo history because we hit errors during the build if the most recent tag isn't in our history for `git describe`
depth: 9999999
lfs_skip_smudge: true
jdk:
- openjdk8
env:
matrix:
- SCALA_VERSION=2.11 TEST_TYPE=cloud UPLOAD=true TESTS_REQUIRE_GCLOUD=true
- SCALA_VERSION=2.11 TEST_TYPE=integration TEST_DOCKER=true TEST_VERBOSITY=minimal
- SCALA_VERSION=2.11 TEST_TYPE=unit TEST_DOCKER=true TEST_VERBOSITY=minimal
- SCALA_VERSION=2.11 TEST_TYPE=variantcalling TEST_DOCKER=true TEST_VERBOSITY=minimal
- SCALA_VERSION=2.11 TEST_TYPE=conda TEST_DOCKER=true TEST_VERBOSITY=minimal
- SCALA_VERSION=2.11 RUN_CNV_GERMLINE_COHORT_WDL=true TESTS_REQUIRE_GCLOUD=true
- SCALA_VERSION=2.11 RUN_CNV_GERMLINE_CASE_WDL=true TESTS_REQUIRE_GCLOUD=true
- SCALA_VERSION=2.11 RUN_CNV_SOMATIC_WDL=true TESTS_REQUIRE_GCLOUD=true
- SCALA_VERSION=2.11 RUN_M2_WDL=true TESTS_REQUIRE_GCLOUD=true
- SCALA_VERSION=2.11 RUN_CNN_WDL=true TESTS_REQUIRE_GCLOUD=true
- SCALA_VERSION=2.11 TEST_TYPE=wdlGen
global:
#gradle needs this
- TERM=dumb
#limit gradle jvm memory and disable daemon
- GRADLE_OPTS="-Xmx2048m -Dorg.gradle.daemon=false"
#google cloud stuff
- export CLOUDSDK_CONFIG=${PWD}/config
- export CLOUDSDK_CORE_DISABLE_PROMPTS=1
- export GCLOUD_HOME=$HOME/gcloud/google-cloud-sdk/bin
- export CLOUDSDK_PYTHON_SITEPACKAGES=1
- export GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/servicekey.json
#hellbender specific variables
- HELLBENDER_TEST_INPUTS=gs://hellbender/test/resources/
- HELLBENDER_TEST_STAGING=gs://hellbender-test-logs/staging/
- HELLBENDER_TEST_LOGS=/hellbender-test-logs/build_reports/
- HELLBENDER_TEST_PROJECT=broad-dsde-dev
- HELLBENDER_JSON_SERVICE_ACCOUNT_KEY=servicekey.json
#Work around for https://github.com/googleapis/google-cloud-java/issues/5884
- export GOOGLE_CLOUD_PROJECT=$HELLBENDER_TEST_PROJECT
# For cromwell jar download
- export CROMWELL_JAR=$HOME/cromwell-51.jar
- export WOMTOOL_JAR=$HOME/womtool-51.jar
matrix:
fast_finish: true
include:
- jdk: openjdk11
language: java
env: SCALA_VERSION=2.12 TEST_TYPE=integration TEST_VERBOSITY=minimal
- jdk: openjdk11
language: java
env: SCALA_VERSION=2.12 TEST_TYPE=unit TEST_VERBOSITY=minimal
- jdk: openjdk11
language: java
env: SCALA_VERSION=2.12 TEST_TYPE=cloud TESTS_REQUIRE_GCLOUD=true TEST_VERBOSITY=minimal
before_cache:
- rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
- rm -fr $HOME/.gradle/caches/*/plugin-resolution/
cache:
directories:
- $HOME/.gradle/caches/
- $HOME/.gradle/wrapper/
- $HOME/.m2
- $HOME/gcloud/
- $HOME/site-library
before_install:
#skip push builds if there's an associated PR because we don't look at them anway
- if [[ ${TRAVIS_EVENT_TYPE} == push ]];
then
PULL_REQUESTS=$( scripts/travis/check_for_pull_request );
if [[ $( grep -c "commits" <<< ${PULL_REQUESTS} ) -gt 0 ]];
then
travis_terminate 0;
fi;
fi;
- if [[ $TRAVIS_SECURE_ENV_VARS == true ]]; then
echo "Logging in to dockerhub";
docker login --username=$DOCKER_SERVICE_LOGIN --password=$DOCKER_SERVICE_TOKEN;
fi
- REPORT_PATH=${TRAVIS_BRANCH}_${TRAVIS_JOB_NUMBER};
- if [[ $TRAVIS_SECURE_ENV_VARS == true && $TRAVIS_EVENT_TYPE != cron ]]; then
echo "Test report will be written to https://storage.googleapis.com$HELLBENDER_TEST_LOGS$REPORT_PATH/tests/test/index.html";
fi;
- if [[ ${TRAVIS_SECURE_ENV_VARS} == true ]]; then
if [[ ${TESTS_REQUIRE_GCLOUD} == true ]]; then
scripts/travis/install_and_authenticate_to_gcloud.sh;
fi;
openssl aes-256-cbc -K $encrypted_8ebb1ef83f64_key -iv $encrypted_8ebb1ef83f64_iv -in resources_for_CI/github_deploy_key.enc -out ~/.ssh/id_rsa -d;
chmod 600 ~/.ssh/id_rsa;
echo "Testing github authentication:";
ssh -T [email protected];
echo "Done testing github authentication";
fi;
# Download Cromwell jar -- if you change the version, please change the CROMWELL_JAR env variable above, too.
- if [[ $TEST_TYPE == wdlGen || $RUN_CNV_GERMLINE_COHORT_WDL == true || $RUN_CNV_GERMLINE_CASE_WDL == true || $RUN_CNV_SOMATIC_WDL == true || $RUN_M2_WDL == true || $RUN_CNN_WDL == true ]]; then
wget -O $CROMWELL_JAR https://github.com/broadinstitute/cromwell/releases/download/51/cromwell-51.jar;
wget -O $WOMTOOL_JAR https://github.com/broadinstitute/cromwell/releases/download/51/womtool-51.jar;
fi;
# Download git lfs files
- git lfs
- git lfs install
- git lfs fetch
- git lfs checkout
# Disable services enabled by default to free a bit of memory
# http://docs.travis-ci.com/user/database-setup/#MySQL
- sudo /etc/init.d/mysql stop
- sudo /etc/init.d/postgresql stop
install:
- if [[ $TRAVIS_SECURE_ENV_VARS == false && $TEST_TYPE == cloud ]]; then
echo "Can't run cloud tests without keys so don't bother building";
elif [[ $TEST_DOCKER == true ]]; then
echo "Skip the install because we're doing a docker build";
else
./gradlew assemble;
./gradlew installDist;
if [[ $RUN_CNV_GERMLINE_COHORT_WDL == true || $RUN_CNV_GERMLINE_CASE_WDL == true || $RUN_CNV_SOMATIC_WDL == true || $RUN_M2_WDL == true || $RUN_CNN_WDL == true ]]; then
echo "building a shadow jar for the wdl";
./gradlew shadowJar;
elif [[ $TEST_TYPE == cloud ]]; then
echo "building a spark jar for the dataproc tests";
./gradlew sparkjar;
fi;
fi;
script:
# run a basic sanity check to be sure that gatk doesn't explode
# run tests
# Because the docker build for GATK doesn't include the test code or the src files we have to manually collect them into a location to be mounted in for performing tests
- if [[ $TRAVIS_EVENT_TYPE == cron ]]; then
echo "Not running any tests for nightly builds";
elif [[ $TEST_TYPE == wdlGen ]]; then
./gradlew gatkWDLGenValidation;
elif [[ $TRAVIS_SECURE_ENV_VARS == false && $TEST_TYPE == cloud ]]; then
echo "Can't run cloud tests without keys so don't run tests";
elif [[ $RUN_CNV_GERMLINE_COHORT_WDL == true ]]; then
echo "Running CNV germline cohort workflow";
travis_wait 60 bash scripts/cnv_cromwell_tests/germline/run_cnv_germline_workflows.sh COHORT;
elif [[ $RUN_CNV_GERMLINE_CASE_WDL == true ]]; then
echo "Running CNV germline case workflow";
travis_wait 60 bash scripts/cnv_cromwell_tests/germline/run_cnv_germline_workflows.sh CASE;
elif [[ $RUN_CNV_SOMATIC_WDL == true ]]; then
echo "Running CNV somatic workflows";
travis_wait 60 bash scripts/cnv_cromwell_tests/somatic/run_cnv_somatic_workflows.sh;
elif [[ $RUN_M2_WDL == true ]]; then
echo "Deleting some unused files before running M2 WDL...";
rm -Rf src/test/resources/large/VQSR;
echo "Running M2 WDL";
travis_wait 60 sudo bash scripts/m2_cromwell_tests/run_m2_wdl.sh;
elif [[ $RUN_CNN_WDL == true ]]; then
echo "Running CNN WDL";
travis_wait 60 sudo bash scripts/cnn_variant_cromwell_tests/run_cnn_variant_wdl.sh;
elif [[ $TEST_DOCKER == true ]]; then
echo "Building docker image and running appropriate tests..." ;
if [ ${TRAVIS_PULL_REQUEST} != false ]; then
sudo bash build_docker.sh -e FETCH_HEAD -s -u -t ${TRAVIS_PULL_REQUEST};
DOCKER_TAG=FETCH_HEAD;
else
echo ${TRAVIS_COMMIT};
sudo bash build_docker.sh -e ${TRAVIS_COMMIT} -s -u;
DOCKER_TAG=$TRAVIS_COMMIT;
fi;
sudo docker images;
echo ${TEST_TYPE};
sudo ./gradlew clean shadowTestClassJar shadowTestJar;
mkdir ./testJars;
echo $( find ./build/libs/ -name "gatk*test.jar");
cp -r $( find ./build/libs/ -name "gatk*test.jar" ) ./testJars;
cp -r $( find ./build/libs/ -name "gatk*testDependencies.jar" ) ./testJars;
mkdir -p build/reports/;
chmod -R a+w build/reports/;
cp scripts/docker/dockertest.gradle .;
sudo docker run -v $(pwd):/gatkCloneMountPoint:cached -v $(pwd)/testJars:/jars:cached --rm -e "scala.version=${SCALA_VERSION}" -e "TEST_VERBOSITY=minimal" -e "TEST_TYPE=${TEST_TYPE}" -t broadinstitute/gatk:${DOCKER_TAG} bash --init-file /gatk/gatkenv.rc /root/run_unit_tests.sh;
TEST_EXIT_VALUE=$?;
sudo mkdir build/reports/tests/test && sudo cp -rp build/reports/tests/testOnPackagedReleaseJar/* build/reports/tests/test && sudo rm -r build/reports/tests/testOnPackagedReleaseJar;
$( exit ${TEST_EXIT_VALUE} );
else
./gatk PrintReads -I src/test/resources/NA12878.chr17_69k_70k.dictFix.bam -O output.bam;
export GATK_LAUNCH_SCRIPT="$(pwd)/gatk";
travis_wait 50 ./gradlew -Dscala.version=${SCALA_VERSION} jacocoTestReport;
fi;
# This creates and uploads the gatk zip file to the nightly build bucket, only keeping the 10 newest entries
# This also constructs the Docker image and uploads it to https://cloud.docker.com/u/broadinstitute/repository/docker/broadinstitute/gatk-nightly/
- if [[ $TRAVIS_BRANCH == master && $TRAVIS_EVENT_TYPE == cron && $UPLOAD == true ]]; then
$GCLOUD_HOME/gcloud components -q update gsutil;
gsutil ls -l gs://gatk-nightly-builds | grep gatk | sort -r -k 2 | grep -o '\S\+$' | tail -n +11 | xargs -I {} gsutil rm {};
./gradlew bundle;
ZIP_FILE="$(ls build/ | grep '^gatk.*.zip$' | grep -iv python)";
echo "Uploading zip to gs://gatk-nightly-builds/";
$GCLOUD_HOME/gsutil -m cp build/$ZIP_FILE gs://gatk-nightly-builds/"$(date +%Y-%m-%d)"-$ZIP_FILE;
bash build_docker.sh -e ${TRAVIS_COMMIT} -s -u;
DOCKER_TAG=$(docker images -q | head -n 1);
DATE=$(date +%Y-%m-%d);
DOCKER_NIGHTLY_TAG=broadinstitute/gatk-nightly:$DATE-$(git describe)-NIGHTLY-SNAPSHOT;
echo "Pushing Nightly Docker image";
docker tag $DOCKER_TAG $DOCKER_NIGHTLY_TAG;
docker push $DOCKER_NIGHTLY_TAG;
fi;
after_success:
# - bash <(curl -s https://codecov.io/bash)
- if [[ $TRAVIS_BRANCH == master && $UPLOAD == true ]]; then ./gradlew uploadArchives; fi;
# if this is actually a commit to master and not a pull request build into master, then publish master-snapshot
- if [[ $TRAVIS_BRANCH == master && $TRAVIS_PULL_REQUEST == false && $UPLOAD == true ]]; then
git tag master;
./gradlew uploadArchives;
fi;
after_failure:
- test -f hs_err_*.log && cat hs_err_*.log
- dmesg | tail -100
- export FAILED=true
after_script:
- if [[ $TRAVIS_SECURE_ENV_VARS == true && $TRAVIS_EVENT_TYPE != cron ]]; then
if [[ ${TESTS_REQUIRE_GCLOUD} != "true" ]]; then
scripts/travis/install_and_authenticate_to_gcloud.sh;
fi;
$GCLOUD_HOME/gsutil -m cp -z html -z js -z xml -z css -r build/reports/tests gs:/$HELLBENDER_TEST_LOGS$REPORT_PATH/;
echo "See the test report at https://storage.googleapis.com$HELLBENDER_TEST_LOGS$REPORT_PATH/tests/test/index.html";
if [[ ${FAILED} == true && ${TRAVIS_PULL_REQUEST} != false && -d build/reports/tests ]]; then
pip install --user PyGithub;
python scripts/travis/Reporter.py https://storage.googleapis.com$HELLBENDER_TEST_LOGS$REPORT_PATH/tests/test/index.html;
fi
fi;