diff --git a/mango-cli/pom.xml b/mango-cli/pom.xml index 4d29786a6..a49a95a6d 100644 --- a/mango-cli/pom.xml +++ b/mango-cli/pom.xml @@ -121,7 +121,6 @@ org.apache.spark spark-sql_${scala.version.prefix} - provided org.apache.parquet @@ -131,7 +130,6 @@ org.apache.spark spark-core_${scala.version.prefix} - provided org.bdgenomics.adam diff --git a/mango-core/pom.xml b/mango-core/pom.xml index a2ed13d3b..4fc8ebaa7 100644 --- a/mango-core/pom.xml +++ b/mango-core/pom.xml @@ -98,12 +98,10 @@ org.apache.spark spark-sql_${scala.version.prefix} - provided org.apache.spark spark-core_${scala.version.prefix} - provided org.bdgenomics.adam diff --git a/pom.xml b/pom.xml index f8f1730b9..90440d85f 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ 2.11.12 2.11 2.4.1 - 2.1.0 + 2.2.1 -spark2_ 1.0.5 1.1.1 @@ -438,6 +438,12 @@ Spark 2.2.0 release notes. --> 1.8.1 + + + org.apache.parquet + * + + org.apache.parquet diff --git a/scripts/jenkins-test b/scripts/jenkins-test index 8b563e672..d300c0cce 100755 --- a/scripts/jenkins-test +++ b/scripts/jenkins-test @@ -39,8 +39,8 @@ fi set -e -# are we testing for spark 2.1.0? if so, we need to rewrite our poms first -if [ ${SPARK_VERSION} == 2.1.0 ]; +# are we testing for spark 2.2.1? if so, we need to rewrite our poms first +if [ ${SPARK_VERSION} == 2.2.1 ]; then # shouldn't be able to move to spark 2 twice @@ -116,7 +116,7 @@ find . -name "*.bak" -exec rm -f {} \; # run integration tests # prebuilt spark distributions are scala 2.11 for spark 2.x -if [[ ${SPARK_VERSION} == 2.1.0 && ${SCALAVER} == 2.11 ]]; +if [[ ${SPARK_VERSION} == 2.2.1 && ${SCALAVER} == 2.11 ]]; then # make a temp directory @@ -158,7 +158,10 @@ then SPARK=spark-${SPARK_VERSION} # download prepackaged spark assembly - wget -q http://d3kbcqa49mib13.cloudfront.net/${SPARK}-bin-${HADOOP}.tgz + curl \ + -L "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=spark/${SPARK}/${SPARK}-bin-${HADOOP}.tgz" \ + -o ${SPARK}-bin-${HADOOP}.tgz + tar xzvf ${SPARK}-bin-${HADOOP}.tgz export SPARK_HOME=${MANGO_TMP_DIR}/${SPARK}-bin-${HADOOP}