.travis.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

sudo: required

before_cache:
  - sudo chown -R travis:travis $HOME/.m2

cache:
  apt: true
  directories:
    - ${HOME}/.m2
    - submarine-workbench/workbench-web/node
    - submarine-workbench/workbench-web/node_modules
    - submarine-workbench/workbench-web-ng/node
    - submarine-workbench/workbench-web-ng/node_modules

addons:
  apt:
    sources:
      - mysql-5.7-trusty
    packages:
      - mysql-server
      - mysql-client

services:
  - mysql
  # https://docs.travis-ci.com/user/gui-and-headless-browsers/#Using-xvfb-to-Run-Tests-That-Require-a-GUI
  - xvfb

env:
  global:
    # If you need to compile Phadoop-3.1 or Phadoop-3.2, you need to add `!submarine-server/server-submitter/submitter-yarnservice` in EXCLUDE_SUBMARINE
    - SUBMARINE="org.apache.submarine"
    - EXCLUDE_SUBMARINE="!:submarine-all,!:submarine-client,!:submarine-dist"
    - EXCLUDE_SERVER="!:submarine-server-api,!:submarine-server-core,!:submarine-server-rpc"
    - EXCLUDE_CLIENT="!:submarine-client"
    - EXCLUDE_ALL="!:submarine-all"
    - EXCLUDE_WORKBENCH="!:submarine-workbench,!:submarine-workbench-web,!:submarine-workbench-web-ng"
    - EXCLUDE_INTERPRETER="!:submarine-interpreter,!:submarine-interpreter-core,!:submarine-python-interpreter,!:submarine-spark-interpreter"
    - EXCLUDE_SUBMITTER_K8S="!:submarine-submitter-k8s"
    - EXCLUDE_SUBMITTER_YARN="!:submarine-submitter-yarn"
    - EXCLUDE_SUBMITTER="!:submarine-server-submitter,${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_SUBMITTER_YARN}"
    - EXCLUDE_COMMONS="!:submarine-commons-cluster,!:submarine-commons-metastore,!:submarine-commons-rpc,!:submarine-commons-runtime,!:submarine-commons-utils"
    - EXCLUDE_CLOUD="!:submarine-cloud"
    - EXCLUDE_DIST="!:submarine-dist"
    - EXCLUDE_TEST="!:submarine-test,!:submarine-test-e2e,!:submarine-test-k8s"
    - EXCLUDE_SPARK_SECURTITY="!:submarine-spark-security"
    - MOZ_HEADLESS=1

before_install:
  # mysql
  - sudo service mysql restart
  - ./docs/database/init-database.sh
  - ./dev-support/travis/install_external_dependencies.sh
  # protobuf 3.10.1
  - PROTOBUF_VERSION=3.10.1
  - PROTOC_FILENAME=protoc-${PROTOBUF_VERSION}-linux-x86_64.zip
  - pushd /home/travis
  - wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/${PROTOC_FILENAME}
  - unzip ${PROTOC_FILENAME}
  - bin/protoc --version
  - popd

matrix:
  include:
    - name: Test submarine test-k8s
      dist: xenial
      services: docker
      language: java
      jdk: openjdk8
      addons:
        chrome: stable
        firefox: latest
      # Because submarine on k8s uses port 80, it needs to be set `-Durl=http://127.0.0.1`
      env:
        - PROFILE="-Phadoop-2.9"
        - BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="verify -DskipRat -am -Durl=http://127.0.0.1"
        - MODULES=""
        - TEST_MODULES="-pl ${EXCLUDE_SERVER},${EXCLUDE_COMMONS},${EXCLUDE_CLIENT},:submarine-test-k8s"
        - TEST_PROJECTS=""

    - name: Test submarine test-e2e
      language: java
      jdk: openjdk8
      dist: xenial
      addons:
        chrome: stable
        firefox: latest
      env:
        - PROFILE="-Phadoop-2.9"
        - BUILD_FLAG="clean package install -DskipTests"
        - TEST_FLAG="verify -DskipRat -am"
        - TEST_MODULES="-pl org.apache.submarine:submarine-test-e2e"
        - TEST_PROJECTS=""

    - name: Test submarine commons-cluster
      language: java
      jdk: openjdk8
      dist: xenial
      env:
        - PROFILE="-Phadoop-2.9"
        - BUILD_FLAG="clean package install -DskipTests"
        - TEST_FLAG="test -DskipRat -am"
        - MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}"
        - TEST_MODULES="-pl org.apache.submarine:submarine-commons-cluster"
        - TEST_PROJECTS=""

    - name: Test submarine commons-metastore
      language: java
      jdk: openjdk8
      dist: xenial
      env:
        - PROFILE="-Phadoop-2.9"
        - BUILD_FLAG="clean package install -DskipTests"
        - TEST_FLAG="test -DskipRat -am"
        - MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}"
        - TEST_MODULES="-pl org.apache.submarine:submarine-commons-metastore"
        - TEST_PROJECTS=""

    - name: Test submarine commons-rpc
      language: java
      jdk: openjdk8
      dist: xenial
      env:
        - PROFILE="-Phadoop-2.9"
        - BUILD_FLAG="clean package install -DskipTests"
        - TEST_FLAG="test -DskipRat -am"
        - MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}"
        - TEST_MODULES="-pl org.apache.submarine:submarine-commons-rpc"
        - TEST_PROJECTS=""

    - name: Test submarine commons-runtime
      language: java
      jdk: openjdk8
      dist: xenial
      env:
        - PROFILE="-Phadoop-2.9"
        - BUILD_FLAG="clean package install -DskipTests"
        - TEST_FLAG="test -DskipRat -am"
        - MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}"
        - TEST_MODULES="-pl org.apache.submarine:submarine-commons-runtime"
        - TEST_PROJECTS=""

    - name: Test submarine commons-unixusersync
      language: java
      jdk: openjdk8
      dist: xenial
      env:
        - PROFILE="-Phadoop-2.9"
        - BUILD_FLAG="clean package install -DskipTests"
        - TEST_FLAG="test -DskipRat -am"
        - MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}"
        - TEST_MODULES="-pl org.apache.submarine:submarine-commons-unixusersync"
        - TEST_PROJECTS=""

    - name: Test submarine server
      language: java
      jdk: openjdk8
      dist: xenial
      env:
        - PROFILE="-Phadoop-2.9"
        - BUILD_FLAG="clean package install -DskipTests"
        - TEST_FLAG="test -DskipRat -am"
        - MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}"
        - TEST_MODULES="-pl ${EXCLUDE_COMMONS},org.apache.submarine:submarine-server-core"
        - TEST_PROJECTS=""

    - name: Test submarine submitter on hadoop-2.9 (default)
      language: java
      jdk: openjdk8
      dist: xenial
      env:
        - PROFILE="-Phadoop-2.9"
        - BUILD_FLAG="clean package install -DskipTests -DskipRat"
        - TEST_FLAG="test -DskipRat -am"
        - MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}"
        - TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER},${EXCLUDE_SPARK_SECURTITY}"
        - TEST_PROJECTS=""

    - name: Test submarine submitter on hadoop-2.10
      language: java
      jdk: openjdk8
      dist: xenial
      env:
        - PROFILE="-Phadoop-2.10"
        - BUILD_FLAG="clean package install -DskipTests -DskipRat"
        - TEST_FLAG="test -DskipRat -am"
        - MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}"
        - TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER},${EXCLUDE_SPARK_SECURTITY}"
        - TEST_PROJECTS=""

    - name: Test submarine submitter on hadoop-3.1
      language: java
      jdk: openjdk8
      dist: xenial
      env:
        - PROFILE="-Phadoop-3.1"
        - BUILD_FLAG="clean package install -DskipTests -DskipRat"
        - TEST_FLAG="test -DskipRat -am"
        - MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}"
        - TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER},${EXCLUDE_SPARK_SECURTITY}"
        - TEST_PROJECTS=""

    - name: Test submarine submitter on hadoop-3.2
      language: java
      jdk: openjdk8
      dist: xenial
      env:
        - PROFILE="-Phadoop-3.2"
        - BUILD_FLAG="clean package install -DskipTests -DskipRat"
        - TEST_FLAG="test -DskipRat -am"
        - MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}"
        - TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER},${EXCLUDE_SPARK_SECURTITY}"
        - TEST_PROJECTS=""

    - name: Test submarine interpreter
      language: java
      jdk: "openjdk8"
      dist: xenial
      env: PYTHON="3" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')" TEST_MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')" TEST_PROJECTS=""

    - name: Test submarine workbench-web VUE
      language: node_js
      node_js:
        - 10.15.0
      before_install:
        - cd submarine-workbench/workbench-web
      install:
        - yarn install
      script:
        - yarn
        - yarn run lint --no-fix && yarn run build
      env: NAME="Build workbench-web"

    - name: Test submarine workbench-web Angular
      language: node_js
      node_js:
        - "10"
      before_install:
        - cd submarine-workbench/workbench-web-ng
      before_script:
        npm install
      addons:
        chrome: stable
        firefox: latest
      script:
        - npm run test -- --no-watch --no-progress --browsers=ChromeHeadlessCI
        - npm run test -- --no-watch --no-progress --browsers=FirefoxHeadless
        - npm run webdriver
        - npm run e2e -- --protractor-config=e2e/protractor-ci.conf.js
      env:
        - BUILD_FLAG="clean package -DskipTests"
        - TEST_FLAG="test -DskipRat -am"
        - MODULES="-pl org.apache.submarine:submarine-workbench-web-ng"
        - TEST_MODULES="-pl org.apache.submarine:submarine-workbench-web-ng"
        - TEST_PROJECTS=""

    - name: Test submarine spark security with spark 2.3 and ranger 1.0
      language: scala
      jdk: openjdk8
      env:
        - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
        - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true"
        - TEST_FLAG=$BUILD_FLAG
        - PROFILE="-Pspark-2.3 -Pranger-1.0"
        - MODULES="-pl :submarine-spark-security"

    - name: Test submarine spark security with spark 2.3 and ranger 1.1
      language: scala
      jdk: openjdk8
      env:
        - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
        - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true"
        - TEST_FLAG=$BUILD_FLAG
        - PROFILE="-Pspark-2.3 -Pranger-1.1"
        - MODULES="-pl :submarine-spark-security"

    - name: Test submarine spark security with spark 2.3 and ranger 1.2
      language: scala
      jdk: openjdk8
      env:
        - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
        - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true"
        - TEST_FLAG=$BUILD_FLAG
        - PROFILE="-Pspark-2.3 -Pranger-1.2"
        - MODULES="-pl :submarine-spark-security"

    - name: Test submarine spark security with spark 2.3 and ranger 2.0
      language: scala
      jdk: openjdk8
      env:
        - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
        - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true"
        - TEST_FLAG=$BUILD_FLAG
        - PROFILE="-Pspark-2.3 -Pranger-2.0"
        - MODULES="-pl :submarine-spark-security"

    - name: Test submarine spark security with spark 2.4 and ranger 1.0
      language: scala
      jdk: openjdk8
      env:
        - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
        - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true"
        - TEST_FLAG=$BUILD_FLAG
        - PROFILE="-Pspark-2.4 -Pranger-1.0"
        - MODULES="-pl :submarine-spark-security"

    - name: Test submarine spark security with spark 2.4 and ranger 1.1
      language: scala
      jdk: openjdk8
      env:
        - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
        - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true"
        - TEST_FLAG=$BUILD_FLAG
        - PROFILE="-Pspark-2.4 -Pranger-1.1"
        - MODULES="-pl :submarine-spark-security"

    - name: Test submarine spark security with spark 2.4 and ranger 1.2
      language: scala
      jdk: openjdk8
      env:
        - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
        - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true"
        - TEST_FLAG=$BUILD_FLAG
        - PROFILE="-Pspark-2.4 -Pranger-1.2"
        - MODULES="-pl :submarine-spark-security"

    - name: Test submarine spark security with spark 2.4 and ranger 2.0
      language: scala
      jdk: openjdk8
      env:
        - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
        - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true"
        - TEST_FLAG=$BUILD_FLAG
        - PROFILE="-Pspark-2.4 -Pranger-2.0"
        - MODULES="-pl :submarine-spark-security"
install:
  - mvn --version
  - echo ">>> mvn $BUILD_FLAG $MODULES $PROFILE -B"
  - mvn $BUILD_FLAG $MODULES $PROFILE -B

before_script:
  # display info log for debugging
  - if [[ -n $TEST_MODULES ]]; then echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=info'" > ~/.mavenrc; fi

script:
  - if [[ -n $TEST_MODULES ]]; then export MODULES="${TEST_MODULES}"; fi
  - echo ">>> mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS"
  - mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS

after_success:
  - echo "Travis exited with ${TRAVIS_TEST_RESULT}"

after_failure:
  - echo "Travis exited with ${TRAVIS_TEST_RESULT}"
  - find . -name rat.txt | xargs cat
  - cat logs/*