jenkinsfile

def MVN               = "mvn -B"
def MVN_OPTS          = "export MAVEN_OPTS=\"-Xms4g -Xmx8g -XX:MaxDirectMemorySize=2048m\""
def MAVEN_SKIP        = "-Danimal.sniffer.skip=true -Dcheckstyle.skip=true -Ddruid.console.skip=true -Denforcer.skip=true -Dforbiddenapis.skip=true -Dmaven.javadoc.skip=true -Dpmd.skip=true -Dspotbugs.skip=true"
def MAVEN_SKIP_TESTS  = "-DskipTests -Djacoco.skip=true"
def M2_CACHE_IMAGE    = "018895040333.dkr.ecr.us-east-1.amazonaws.com/druid-m2-cache"
def BUILD_CACHE_IMAGE = "018895040333.dkr.ecr.us-east-1.amazonaws.com/druid-build-cache"
def AWS_CREDS_ID      = "aws_s3_access"
def AZURE_CREDS_ID    = "azure_credentials"
def BUILD_CACHE_TAG   = env.BUILD_TAG.toLowerCase().replaceAll(~/[^a-z0-9-]/,'')

def dockerInitCleanup = {
    sh script: "for container in \$(docker ps -q); do docker stop \$container; done"
    sh script: "docker container prune -f"
    sh script: "docker volume prune -f"
    sh script: "docker network prune -f"
}

def workspaceCleanup = {
    sh script: "sudo git clean -fdx", label: "Remove everything but files from git"
}

def withArtifactorySettings = { body ->
    withCredentials([file(credentialsId: 'maven-artifactory-settings', variable: 'MVN_SETTINGS_PATH')]) {
        sh script: "ln -sf ${env.MVN_SETTINGS_PATH} ~/.m2/settings.xml"
        try {
            body()
        } finally {
            sh script: "rm -f ~/.m2/settings.xml"
        }
    }
}

def withAWSCredentials = { body ->
    withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'aws_s3_access', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
        body()
    }
}

def withAzureCredentials = { body ->
    withCredentials([usernamePassword(credentialsId: AZURE_CREDS_ID, usernameVariable: 'AZURE_ACCOUNT', passwordVariable: 'AZURE_KEY')]) {
        body()
    }
}

def copyFilesByPattern = { filesPattern, destDir ->
    sh script: """#!/bin/bash -x
        shopt -s globstar
        if  ls ${filesPattern}; then
            for fname in ${filesPattern}; do
                fname_root=\$(echo \${fname} | cut -d "/" -f1)
                if ! [[ "\$fname_root" = "${destDir}" ]]; then
                    mkdir -p \$(dirname -- "${destDir}/\${fname}")
                    cp \$fname ${destDir}/\${fname}
                fi
            done
        fi
    """
}

def inspectException = { body ->
    try {
        body()
    } catch(err) {
        echo "Exception: ${err}"
        def causeClasses = err.causes.collect {it.getClass()}
        echo "Exception causes: ${causeClasses}"
        throw err
    }
}

def retryOnTimeout = { body ->
    while(true) {
        try {
            inspectException {
                body()
            }
            break
        } catch(err) {
            if (err instanceof org.jenkinsci.plugins.workflow.steps.FlowInterruptedException &&
                err.causes.collect {it.getClass()}.contains(org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout)) {
                continue
            } else {
                throw err
            }
        }
    }
}

def getDiffBranchName = {
    if (env.CHANGE_ID && env.CHANGE_TARGET && env.CHANGE_BRANCH){
        return env.CHANGE_TARGET
    }
    else {
        return env.BRANCH_NAME
    }
}

def artifactsWrapper = { body ->
    try {
        body()
    } finally {
        def stageArtifactsDirPath = "stage_${env.STAGE_NAME.replaceAll(~/[^A-Za-z0-9_-]/,'_')}"
        sh script: "sudo mkdir -p /root/shared"
        sh script: "mkdir -p ${stageArtifactsDirPath}"
        // copy logs
        sh script: "sudo bash -c '[ -d \"/root/shared/logs\" ] && cp -R /root/shared/logs ${stageArtifactsDirPath}/ || true'"
        sh script: "sudo bash -c '[ -d \"/root/shared/tasklogs\" ] && cp -R /root/shared/tasklogs ${stageArtifactsDirPath}/ || true'"
        // copy test reports
        copyFilesByPattern("**/target/surefire-reports/*.xml", stageArtifactsDirPath)
        copyFilesByPattern("**/target/failsafe-reports/*.xml", stageArtifactsDirPath)
        // copy top-level jacoco reports
        copyFilesByPattern("**/target/*.exec", stageArtifactsDirPath)
        copyFilesByPattern("**/jacoco/*.html", stageArtifactsDirPath)
        copyFilesByPattern("**/jacoco/*.xml", stageArtifactsDirPath)
        copyFilesByPattern("**/jacoco/*.csv", stageArtifactsDirPath)
        copyFilesByPattern("**/jacoco/jacoco-resources/*", stageArtifactsDirPath)
        // copy detailed jacoco reports
        copyFilesByPattern("**/jacoco/**/*.html", stageArtifactsDirPath)
        copyFilesByPattern("**/jacoco/**/*.xml", stageArtifactsDirPath)
        copyFilesByPattern("**/jacoco/**/*.csv", stageArtifactsDirPath)
        // fixate artifacts
        archiveArtifacts artifacts: "${stageArtifactsDirPath}/**", allowEmptyArchive: true
    }
}

def heavyNode = { body ->
    retryOnTimeout {
        node('jenkinsOnDemand') {
            artifactsWrapper {
                body()
            }
        }
    }
}

def lightweightNode = { body ->
    retryOnTimeout {
        node('jenkinsOnDemandMultiExec') {
            artifactsWrapper {
                body()
            }
        }
    }
}

def withGitCredentials = { body ->
    withCredentials([
        usernamePassword(
            credentialsId: scm.getUserRemoteConfigs()[0].getCredentialsId(),
            passwordVariable: 'GIT_PASSWORD',
            usernameVariable: 'GIT_USERNAME')
        ]) {
            body()
    }
}


def ecrLogin = { imageURL ->
    def registryHost = (imageURL =~ /^[0-9a-z.-]+/).getAt(0)
    withAWSCredentials {
        sh script: """
            PATH=/home/jenkins/bin:/home/jenkins/.local/bin:\$PATH \
            aws ecr get-login-password --region us-east-1 | \
            docker login --username AWS --password-stdin ${registryHost}
        """
    }
}

def withM2Cache = { body ->
    def destination = "${env.WORKSPACE}@tmp/.m2Cache"
    def buildContext = "${env.WORKSPACE}@tmp/.m2CacheBuildContext"
    ecrLogin(M2_CACHE_IMAGE)
    timeout(time: 15, unit: 'MINUTES') {
        sh script: """
            docker pull ${M2_CACHE_IMAGE}:latest && \
            rm -rf ${buildContext} && mkdir -p ${buildContext} && \
            echo "FROM ${M2_CACHE_IMAGE}:latest" > ${buildContext}/Dockerfile && \
            DOCKER_BUILDKIT=1 docker build --output type=local,dest=${destination} ${buildContext} || \
            { echo "Probably ${M2_CACHE_IMAGE}:latest doesn't exist yet, so using empty dir instead" && mkdir -p ${destination}/.m2; }
            sudo chmod a+w ${destination}/.m2
        """
    }
    body(destination)
}

def uploadM2Cache = { cacheDirPath ->
    def dockerfilePath = "${cacheDirPath}/m2CacheDockerfile"
    ecrLogin(M2_CACHE_IMAGE)
    sh script: """
        cd ${cacheDirPath}
        echo "FROM scratch" > ${dockerfilePath}
        echo "ADD .m2 /.m2" >> ${dockerfilePath}
        docker build -t ${M2_CACHE_IMAGE}:${BUILD_CACHE_TAG} -f ${dockerfilePath} .
        docker tag ${M2_CACHE_IMAGE}:${BUILD_CACHE_TAG} ${M2_CACHE_IMAGE}:latest
        docker push ${M2_CACHE_IMAGE}:${BUILD_CACHE_TAG}
        docker push ${M2_CACHE_IMAGE}:latest
    """
}

def downloadBuildCache = {
    def buildContext = "${env.WORKSPACE}@tmp/.cacheBuildContext"
    ecrLogin(BUILD_CACHE_IMAGE)
    timeout(time: 15, unit: 'MINUTES') {
        sh script: """
            rm -rf ${buildContext}
            mkdir -p ${buildContext}
            echo "FROM ${BUILD_CACHE_IMAGE}:${BUILD_CACHE_TAG}" > ${buildContext}/Dockerfile
            DOCKER_BUILDKIT=1 docker build --output type=local,dest=\$(pwd) ${buildContext}
        """
    }
}


def buildGenerateLicenseImage = { fromImage ->
    writeFile file: "${env.WORKSPACE}@tmp/GenerateLicenseImageDockerfile", text: """\
FROM ${fromImage}
RUN apt-get update -y && apt-get install python3 python3-pip python3-setuptools -y
RUN pip3 install wheel pyyaml
"""

    def imageName = "${fromImage}-license-${env.STAGE_NAME.replaceAll(~/[^a-z0-9]/,'')}${env.BUILD_ID}"
    docker.build(imageName, "${env.WORKSPACE}@tmp -f ${env.WORKSPACE}@tmp/GenerateLicenseImageDockerfile")
    return imageName
}

def buildModuleTestImage = { fromImage ->
    writeFile file: "${env.WORKSPACE}@tmp/ProcessingModuleImageDockerfile", text: """\
FROM ${fromImage}
RUN curl -sL https://deb.nodesource.com/setup_lts.x | bash - && \
    apt-get install -y nodejs
RUN chmod 4755 /bin/dmesg
"""

    def imageName = "${fromImage}-processing-${env.STAGE_NAME.replaceAll(~/[^a-z0-9]/,'')}${env.BUILD_ID}"
    docker.build(imageName, "${env.WORKSPACE}@tmp -f ${env.WORKSPACE}@tmp/ProcessingModuleImageDockerfile")
    return imageName
}


def buildDockerBinaryImage = { fromImage ->
    writeFile file: "${env.WORKSPACE}@tmp/DockerBinaryImageDockerfile", text: """\
FROM ${fromImage}
ARG DOCKER_VERSION=19.03.8
RUN set -ex \
    && curl -fsSLO https://download.docker.com/linux/static/stable/x86_64/docker-\${DOCKER_VERSION}.tgz \
    && mv docker-\${DOCKER_VERSION}.tgz docker.tgz \
    && tar xzvf docker.tgz \
    && mv docker/docker /usr/local/bin/docker \
    && rm -r docker docker.tgz
RUN curl -s -L \
    "https://github.com/docker/compose/releases/download/1.26.2/docker-compose-\$(uname -s)-\$(uname -m)" \
    -o /usr/local/bin/docker-compose \
    && chmod +x /usr/local/bin/docker-compose
RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash
RUN curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
    && unzip awscliv2.zip \
    && ./aws/install \
    && rm awscliv2.zip \
    && rm -rf aws
"""
    def imageName = "${fromImage}-dbin-${env.STAGE_NAME.replaceAll(~/[^a-z0-9]/,'')}${env.BUILD_ID}"
    docker.build(imageName, "${env.WORKSPACE}@tmp -f ${env.WORKSPACE}@tmp/DockerBinaryImageDockerfile")
    return imageName
}


def moduleTest = {
    def diffBranchName = getDiffBranchName()
    sh script: """#!/bin/bash -ex

        /bin/echo -e "#!/bin/sh\\ncase \"\\\$1\" in\\nUsername*) echo \"\\\$GIT_USERNAME\" ;;\\nPassword*) echo \"\\\$GIT_PASSWORD\" ;;\\nesac\\n" | tee -a /bin/askpass
        chmod +x /bin/askpass
        export GIT_ASKPASS=/bin/askpass

        unset _JAVA_OPTIONS
        export BUILD_STATUS=0

        MAVEN_OPTS='-Xmx800m' ${MVN} test -pl \${MAVEN_PROJECTS} \
        ${MAVEN_SKIP} -Dremoteresources.skip=true -Ddruid.generic.useDefaultValueForNull=\${DRUID_USE_DEFAULT_VALUE_FOR_NULL} || \
        export BUILD_STATUS=1

        sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
        free -m
        ${MVN} -pl \${MAVEN_PROJECTS} jacoco:report

        export DIFF_BRANCH_NAME=${diffBranchName}
        git remote set-branches --add origin \${DIFF_BRANCH_NAME} && git fetch
        all_files="\$(git diff --name-only origin/\${DIFF_BRANCH_NAME}...HEAD | grep "\\.java\$" || [[ \$? == 1 ]])"

        for f in \${all_files}; do
            echo \$f
        done

        if [[ "\${MAVEN_PROJECTS}" = \\!* ]]; then
          regex="\${MAVEN_PROJECTS:1}";
          regex="^\${regex//,\\!/\\\\|^}";
          project_files="\$(echo "\${all_files}" | grep -v "\${regex}" || [[ \$? == 1 ]])";
        else
          regex="^\${MAVEN_PROJECTS//,/\\\\|^}";
          project_files="\$(echo "\${all_files}" | grep "\${regex}" || [[ \$? == 1 ]])";
        fi

        for f in \${project_files}; do
            echo \$f
        done

        if [ -n "\${project_files}" ]; then
            { for i in 1 2 3; do npm install @connectis/diff-test-coverage@1.5.3 && break || sleep 15; done } && \
            git diff origin/\${DIFF_BRANCH_NAME}...HEAD -- \${project_files} | \
            node_modules/.bin/diff-test-coverage \
              --coverage "**/target/site/jacoco/jacoco.xml" \
              --type jacoco \
              --line-coverage 50 \
              --branch-coverage 50 \
              --function-coverage 0 \
              --log-template "coverage-lines-complete" \
              --log-template "coverage-files-complete" \
              --log-template "totals-complete" \
              --log-template "errors" \
              -- || \
            { printf "\\n\\n****FAILED****\\nDiff code coverage check failed. To view coverage report, run 'mvn clean test jacoco:report' and open 'target/site/jacoco/index.html'\\nFor more details on how to run code coverage locally, follow instructions here - https://github.com/apache/druid/blob/master/dev/code-review/code-coverage.md#running-code-coverage-locally\\n\\n" && false; }
        fi

        exit \$BUILD_STATUS
    """
}

def mavenCheck = { stageName, body ->
    stage(stageName) {
        lightweightNode {
            checkout scm
            workspaceCleanup()
            downloadBuildCache()
            withM2Cache { cacheDir ->
                docker.image('maven:3.6.3-jdk-8').inside(
                    "--memory=8g --memory-reservation=4g -e HOME=/tmp -e _JAVA_OPTIONS=-Duser.home=/tmp -v ${cacheDir}/.m2:/tmp/.m2") {
                    withArtifactorySettings {
                        body()
                    }
                }
            }
        }
    }
}

def testModule = { stageName, jdkVersion, envMap ->
    stage(stageName) {
        lightweightNode {
            checkout scm
            workspaceCleanup()
            downloadBuildCache()
            withM2Cache { cacheDir ->
                docker.image(buildModuleTestImage("maven:3.6.3-jdk-${jdkVersion}")).inside(
                    "--memory=8g --memory-reservation=4g --privileged -v ${cacheDir}/.m2:/root/.m2 -u root:root") {
                    withArtifactorySettings {
                        withGitCredentials {
                            withEnv(envMap) {
                                moduleTest()
                            }
                        }
                    }
                }
            }
        }
    }
}


def testModuleWithDocker = { stageName, jdkVersion, envMap ->
    stage(stageName) {
        heavyNode {
            dockerInitCleanup()
            withM2Cache { cacheDir ->
                def moduleTestImage = buildModuleTestImage("maven:3.6.3-jdk-${jdkVersion}")
                def dockerBinaryImage = buildDockerBinaryImage(moduleTestImage)
                checkout scm
                workspaceCleanup()
                downloadBuildCache()
                docker.image(dockerBinaryImage).inside("""\
                    --memory=8g \
                    --memory-reservation=4g \
                    --privileged \
                    -v ${cacheDir}/.m2:/root/.m2 \
                    -v /var/run/docker.sock:/var/run/docker.sock \
                    --net host \
                    -u root:root \
                    """) {
                    withArtifactorySettings {
                        withGitCredentials {
                            withEnv(envMap) {
                                moduleTest()
                            }
                        }
                    }
                }
            }
        }
    }
}

def integrationTestsWrapper = { stageName, jdkVersion, envMap, body ->
    stage(stageName) {
        heavyNode {
            dockerInitCleanup()
            sh script: "sudo rm -rf /root/shared"
            sh script: "sudo mkdir -p /root/shared"
            withM2Cache { cacheDir ->
                checkout scm
                workspaceCleanup()
                downloadBuildCache()
                docker.image(buildDockerBinaryImage("maven:3.6.3-jdk-${jdkVersion}")).inside("""\
                    -v ${cacheDir}/.m2:/root/.m2 \
                    -v /var/run/docker.sock:/var/run/docker.sock \
                    -v /root/shared:/root/shared \
                    -e DOCKER_IP=127.0.0.1 \
                    --net host \
                    -u root:root \
                    """) {
                    withArtifactorySettings {
                        withEnv(envMap) {
                            sh script: """
                                echo "deb http://security.debian.org/debian-security jessie/updates main" >> /etc/apt/sources.list
                                apt-get update -y
                                apt-get install dnsutils -y
                                apt-get install openssl=1.0.1t-1+deb8u12 -y --allow-downgrades
                            """
                            try {
                                body()
                            } finally {
                                sh script: """#!/bin/bash -ex
                                    if  ls ~/shared/logs/*.log; then
                                        for v in ~/shared/logs/*.log; do
                                            echo \$v logtail ========================
                                            tail -100 \$v
                                        done
                                    fi
                                    docker ps -a
                                    for v in broker middlemanager overlord router coordinator historical; do
                                        echo \$v dmesg ========================
                                        docker ps --format {{.Names}} | grep -q druid-\$v && docker exec druid-\$v sh -c 'dmesg | tail -3' || true
                                    done
                                """
                            }
                        }
                    }
                }
            }
        }
    }
}

def integrationTests = { stageName, jdkVersion, envMap ->
    integrationTestsWrapper(stageName, jdkVersion, envMap) {
        sh script: """
            export BUILD_STATUS=0
            ${MVN} verify -pl integration-tests -P integration-tests \${TESTNG_GROUPS} \${JVM_RUNTIME} ${MAVEN_SKIP} || \
            export BUILD_STATUS=1
            ${MVN} -pl integration-tests jacoco:report
            exit \$BUILD_STATUS
        """
    }
}

def s3DeepStorageTests = { stageName, jdkVersion, envMap ->
    integrationTestsWrapper(stageName, jdkVersion, envMap) {
        withAWSCredentials {
            def cloudpath = UUID.randomUUID().toString()
            sh script: """
                aws s3 sync \
                ${WORKSPACE}/integration-tests/src/test/resources/data/batch_index/ s3://druid-qa/${cloudpath} \
                --exclude "*" --include "wikipedia_index_data*.json"
            """
            writeFile file: "jenkins/s3-config", text: "druid_storage_type=s3\ndruid_storage_bucket=druid-qa\ndruid_storage_baseKey=${cloudpath}\ndruid_s3_accessKey=${AWS_ACCESS_KEY_ID}\ndruid_s3_secretKey=${AWS_SECRET_ACCESS_KEY}\nAWS_REGION=us-east-1\ndruid_extensions_loadList=[\"druid-s3-extensions\",\"druid-hdfs-storage\"]"
            try {
                sh script: """
                    ${MVN_OPTS}
                    ${MVN} verify -P integration-tests -pl integration-tests \
                    -Dgroups=s3-deep-storage -Doverride.config.path=${WORKSPACE}/jenkins/s3-config \${JVM_RUNTIME} -Ddruid.test.config.cloudBucket=druid-qa -Ddruid.test.config.cloudPath=${cloudpath}/ -Dstart.hadoop.docker=true \
                    -ff ${MAVEN_SKIP} -Djacoco.skip=true
                """, label: "s3-deep-storage with ${env.JVM_RUNTIME}"
            }
            finally {
                sh script: "aws s3 rm s3://druid-qa/${cloudpath}  --recursive"
            }
        }
    }
}

def kinesisDeepStorageTests = { stageName, jdkVersion, envMap ->
    integrationTestsWrapper(stageName, jdkVersion, envMap) {
        lock('awsResource') {
            withAWSCredentials {
                writeFile file: "jenkins/kinesis-config", text: "druid_kinesis_accessKey=${AWS_ACCESS_KEY_ID}\ndruid_kinesis_secretKey=${AWS_SECRET_ACCESS_KEY}\nAWS_REGION=us-east-1\ndruid_extensions_loadList=[\"druid-kinesis-indexing-service\"]"
                sh script: """
                    ${MVN_OPTS}
                    ${MVN} verify -P integration-tests -pl integration-tests \
                    -Dgroups=kinesis-index -Doverride.config.path=${WORKSPACE}/jenkins/kinesis-config \${JVM_RUNTIME} -Ddruid.test.config.streamEndpoint=kinesis.us-east-1.amazonaws.com -Dstart.hadoop.docker=true \
                    -ff ${MAVEN_SKIP} -Djacoco.skip=true
                """, label: "kinesis-deep-storage with ${env.JVM_RUNTIME}"
            }
        }
    }
}

def azureDeepStorageTests = { stageName, jdkVersion, envMap ->
    integrationTestsWrapper(stageName, jdkVersion, envMap) {
        withAzureCredentials {
            def containerName = UUID.randomUUID().toString()
            sh script: """
                az storage container create -n ${containerName} \
                --public-access blob \
                --account-name ${AZURE_ACCOUNT} --account-key ${AZURE_KEY}
            """
            writeFile file: "jenkins/azure-config", text: "druid_storage_type=azure\ndruid_azure_account=$AZURE_ACCOUNT\ndruid_azure_key=$AZURE_KEY\ndruid_azure_container=${containerName}\ndruid_extensions_loadList=[\"druid-azure-extensions\",\"druid-hdfs-storage\"]"
            try {
                sh script: """
                    az storage blob upload-batch \
                    --account-name ${AZURE_ACCOUNT} --account-key ${AZURE_KEY} \
                    -d ${containerName} \
                    --source ${WORKSPACE}/integration-tests/src/test/resources/data/batch_index/ --pattern "wikipedia_index_data*.json"
                """
                sh script: """
                    ${MVN_OPTS}
                    ${MVN} verify -P integration-tests -pl integration-tests \
                    -Dgroups=azure-deep-storage -Doverride.config.path=${WORKSPACE}/jenkins/azure-config \${JVM_RUNTIME} -Ddruid.test.config.cloudBucket=${containerName} -Ddruid.test.config.cloudPath= -Dstart.hadoop.docker=true \
                    -ff ${MAVEN_SKIP} -Djacoco.skip=true
                """, label: "azure-deep-storage with ${env.JVM_RUNTIME}"
            }
            finally {
                sh script: """
                     az storage container delete -n ${containerName}\
                     --account-name ${AZURE_ACCOUNT} --account-key ${AZURE_KEY}
                """
            }
        }
    }
}

def hdfsDeepStorageTests = { stageName, jdkVersion, envMap ->
    integrationTestsWrapper(stageName, jdkVersion, envMap) {
        writeFile file: "${WORKSPACE}/jenkins/hdfs-config", text: "druid_storage_type=hdfs\n" +
            "druid_storage_storageDirectory=/druid/segments\n" +
            "druid_extensions_loadList=[\"druid-hdfs-storage\", \"druid-parquet-extensions\", \"druid-orc-extensions\"]\n" +
            "druid_indexer_logs_type=hdfs\n" +
            "druid_indexer_logs_directory=/druid/indexing-logs"
        sh script: """
            ${MVN_OPTS}
            ${MVN} verify -P integration-tests -pl integration-tests \
            -Dgroups=hdfs-deep-storage -Dstart.hadoop.docker=true \${JVM_RUNTIME} -Doverride.config.path=${WORKSPACE}/jenkins/hdfs-config -Dextra.datasource.name.suffix='' -Dit.test=ITHdfsToHdfsParallelIndexTest \
            -ff ${MAVEN_SKIP} -Djacoco.skip=true
        """, label: "hdfs-deep-storage with ${env.JVM_RUNTIME}"
    }
}

def packagingCheck = { jdkVersion ->
    stage("(openjdk${jdkVersion}) packaging check") {
        lightweightNode {
            withM2Cache { cacheDir ->
                checkout scm
                workspaceCleanup()
                docker.image(buildGenerateLicenseImage("maven:3.6.3-jdk-${jdkVersion}")).inside(
                    "--memory=8g --memory-reservation=4g -v ${cacheDir}/.m2:/root/.m2 -u root:root") {
                    withArtifactorySettings {
                        sh script: """
                            MAVEN_OPTS='-Xmx3000m' ${MVN} -e clean install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end \
                            -pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Ddruid.console.skip=false -T1C
                        """
                    }
                }
            }
        }
    }
}

stage('Maven install') {
    heavyNode {
        withM2Cache { cacheDir ->
            checkout scm
            workspaceCleanup()
            docker.image('maven:3.6.3-jdk-8').inside("""\
                --memory=8g --memory-reservation=4g \
                -e HOME=/tmp -e _JAVA_OPTIONS=-Duser.home=/tmp -v ${cacheDir}/.m2:/tmp/.m2 \
                """ ) {
                withArtifactorySettings {
                    sh script: "${MVN_OPTS} && ${MVN} clean install -q -ff ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T 1C", label: "Maven install"
                }
            }
            uploadM2Cache(cacheDir)

            def contextPath = "${env.WORKSPACE}@tmp/.buildcachecontext"
            def dockerfileName = "BuildCacheDockerfile"

            sh script: """
                rm -rf ${contextPath}
                mkdir -p ${contextPath}
                git status --ignored --porcelain | egrep '^\\!\\!' | grep -v web-console | sed 's|/\$||' | awk -F'\\!\\! ' '{print \$2}'  | tee -a ${contextPath}/artifacts.txt
                for aname in \$(cat ${contextPath}/artifacts.txt | grep -F '/'); do mkdir -p \$(dirname -- "${contextPath}/root/\${aname}"); done
                cat ${contextPath}/artifacts.txt | xargs -I BUILDSOURCE mv BUILDSOURCE ${contextPath}/root/BUILDSOURCE
                cd ${contextPath}
                echo "FROM scratch" > ${dockerfileName}
                echo "COPY root/ /" >> ${dockerfileName}
                docker build  -t ${BUILD_CACHE_IMAGE}:${BUILD_CACHE_TAG} -f ${dockerfileName} .
                docker push ${BUILD_CACHE_IMAGE}:${BUILD_CACHE_TAG}
            """
        }
    }
}

stage("Checks") {
    parallel "animal sniffer checks": {
        mavenCheck("animal sniffer checks") {
            sh script: "${MVN} animal-sniffer:check --fail-at-end", label: "animal sniffer"
        }
    },
    "checkstyle": {
        mavenCheck("checkstyle") {
            sh script: "${MVN} checkstyle:checkstyle --fail-at-end",label: "checkstyle"
        }
    },
    "enforcer checks": {
        mavenCheck("enforcer checks") {
            sh script: "${MVN} enforcer:enforce --fail-at-end", label: "enforcer checks"
        }
    },
    "forbidden api checks": {
        mavenCheck("forbidden api checks") {
            sh script: "${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end", label: "forbidden api checks"
        }
    },
    "pmd checks": {
        mavenCheck("pmd checks") {
            sh script: "${MVN} pmd:check --fail-at-end", label: "pmd checks"
        }
    },
    "spotbugs checks": {
        mavenCheck("spotbugs checks") {
            sh script: "${MVN} spotbugs:check --fail-at-end -pl '!benchmarks' -T 1C", label: "spotbugs checks"
        }
    },
    "license checks": {
        stage('license checks') {
            lightweightNode {
                withM2Cache { cacheDir ->
                    checkout scm
                    downloadBuildCache()
                    docker.image(buildGenerateLicenseImage("maven:3.6.3-jdk-8")).inside(
                        "--memory=8g --memory-reservation=4g -v ${cacheDir}/.m2:/root/.m2 -u root:root") {
                        withArtifactorySettings {
                            sh script: """
                                ${MVN_OPTS}
                                ${MVN} apache-rat:check -Prat --fail-at-end \
                                    -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
                                    -Drat.consoleOutput=true -Drat.numUnapprovedLicenses=100
                                mkdir -p target
                                distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2
                                distribution/bin/check-licenses.py licenses.yaml target/license-reports
                            """, label: "license checks"
                        }
                    }
                }
            }
        }
    },
    "(openjdk8) strict compilation": {
        stage("(openjdk8) strict compilation") {
            lightweightNode {
                withM2Cache { cacheDir ->
                    docker.image('maven:3.6.3-jdk-8').inside(
                        "--memory=8g --memory-reservation=4g -v ${cacheDir}/.m2:/root/.m2 -u root:root") {
                        checkout scm
                        withArtifactorySettings {
                            sh script: "MAVEN_OPTS='-Xmx3000m' ${MVN} clean -Pstrict compile test-compile --fail-at-end \
                                -pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}"
                        }
                    }
                }
            }
        }
    },
    "analyze dependencies": {
        stage("analyze dependencies") {
            lightweightNode {
                withM2Cache { cacheDir ->
                    checkout scm
                    downloadBuildCache()
                    docker.image('maven:3.6.3-jdk-8').inside(
                        "--memory=8g --memory-reservation=4g -v ${cacheDir}/.m2:/root/.m2 -u root:root") {
                        withArtifactorySettings {
                            sh script: """
                                MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true || { echo "
                                The dependency analysis has found a dependency that is either:
                                1) Used and undeclared: These are available as a transitive dependency but should be explicitly
                                added to the POM to ensure the dependency version. The XML to add the dependencies to the POM is
                                shown above.
                                2) Unused and declared: These are not needed and removing them from the POM will speed up the build
                                and reduce the artifact size. The dependencies to remove are shown above.
                                If there are false positive dependency analysis warnings, they can be suppressed:
                                https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html#usedDependencies
                                https://maven.apache.org/plugins/maven-dependency-plugin/examples/exclude-dependencies-from-dependency-analysis.html
                                For more information, refer to:
                                https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html
                                " && false; }
                            """
                        }
                    }
                }
            }
        }
    },
    "intellij inspections": {
        stage("intellij inspections") {
            lightweightNode {
                withM2Cache { cacheDir ->
                    checkout scm
                    downloadBuildCache()
                    sh script: """
                        docker run --rm \
                        -v ${env.WORKSPACE}:/project \
                        -v ${cacheDir}/.m2:/home/inspect/.m2 \
                        ccaominh/intellij-inspect:1.0.0 \
                        /project/pom.xml \
                        /project/.idea/inspectionProfiles/Druid.xml \
                        --levels ERROR \
                        --scope JavaInspectionsScope
                    """
                }
            }
        }
    },
    "(openjdk8) packaging check": {
        packagingCheck(8)
    },
    "(openjdk11) packaging check": {
        packagingCheck(11)
    },
    "(openjdk8) processing module test": {
        testModule("(openjdk8) processing module test", 8, ["MAVEN_PROJECTS=processing", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=true"])
    },
    "(openjdk11) processing module test": {
        testModule("(openjdk11) processing module test", 11, ["MAVEN_PROJECTS=processing", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=true"])
    },
    "(openjdk8) processing module test (SQL Compatibility)": {
        testModule("(openjdk8) processing module test (SQL Compatibility)", 8, ["MAVEN_PROJECTS=processing", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=false"])
    },
    "(openjdk11) processing module test (SQL Compatibility)": {
        testModule("(openjdk11) processing module test (SQL Compatibility)", 11, ["MAVEN_PROJECTS=processing", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=false"])
    },
    "(openjdk8) indexing modules test": {
        testModule("(openjdk8) indexing modules test", 8, ["MAVEN_PROJECTS=indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=true"])
    },
    "(openjdk11) indexing modules test": {
        testModule("(openjdk11) indexing modules test", 11, ["MAVEN_PROJECTS=indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=true"])
    },
    "(openjdk8) indexing modules test (SQL Compatibility)": {
        testModule("(openjdk8) indexing modules test (SQL Compatibility)", 8, ["MAVEN_PROJECTS=indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=false"])
    },
    "(openjdk11) indexing modules test (SQL Compatibility)": {
        testModule("(openjdk11) indexing modules test (SQL Compatibility)", 11, ["MAVEN_PROJECTS=indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=false"])
    },
    "(openjdk8) server modules test": {
        testModule("(openjdk8) server modules test", 8, ["MAVEN_PROJECTS=server", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=true"])
    },
    "(openjdk11) server modules test": {
        testModule("(openjdk11) server modules test", 11, ["MAVEN_PROJECTS=server", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=true"])
    },
    "(openjdk8) server modules test (SQL Compatibility)": {
        testModule("(openjdk8) server modules test (SQL Compatibility)", 8, ["MAVEN_PROJECTS=server", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=false"])
    },
    "(openjdk11) server modules test (SQL Compatibility)": {
        testModule("(openjdk11) server modules test (SQL Compatibility)", 11, ["MAVEN_PROJECTS=server", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=false"])
    },
    "(openjdk8) other modules test": {
        testModuleWithDocker("(openjdk8) other modules test", 8, ["MAVEN_PROJECTS=!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=true"])
    },
    "(openjdk11) other modules test": {
        testModuleWithDocker("(openjdk11) other modules test", 11, ["MAVEN_PROJECTS=!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=true"])
    },
    "(openjdk8) other modules test (SQL Compatibility)": {
        testModuleWithDocker("(openjdk8) other modules test (SQL Compatibility)", 8, ["MAVEN_PROJECTS=!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=false"])
    },
    "(openjdk11) other modules test (SQL Compatibility)": {
        testModuleWithDocker("(openjdk11) other modules test (SQL Compatibility)", 11, ["MAVEN_PROJECTS=!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console", "DRUID_USE_DEFAULT_VALUE_FOR_NULL=false"])
    },
    "web console": {
        stage("web console") {
            lightweightNode {
                withM2Cache { cacheDir ->
                    docker.image(buildModuleTestImage("maven:3.6.3-jdk-8")).inside(
                        "--memory=8g --memory-reservation=4g -v ${cacheDir}/.m2:/root/.m2 -u root:root") {
                        checkout scm
                        withArtifactorySettings {
                            sh script: "${MVN} test -pl 'web-console'"
                        }
                        sh script: """
                            cd web-console
                            for i in 1 2 3; do npm run codecov && break || sleep 15; done
                        """
                    }
                }
            }
        }
    },
    "web console end-to-end test": {
        stage("web console end-to-end test") {
            lightweightNode {
                def moduleTestImage = buildModuleTestImage("maven:3.6.3-jdk-8")
                def generateLicenseImage = buildGenerateLicenseImage(moduleTestImage)

                docker.image(generateLicenseImage).inside("--memory=8g --memory-reservation=4g -u root:root") {
                    checkout scm
                    sh script: """
                        web-console/script/druid build
                        web-console/script/druid start &
                        sleep 10
                    """
                    try {
                        sh script: "cd web-console && npm run test-e2e"
                    } finally {
                        sh script: "web-console/script/druid stop"
                    }
                }
            }
        }
    },
    "docs": {
        stage("docs") {
            lightweightNode {
                docker.image('node:10.21.0-jessie-slim').inside("--memory=8g --memory-reservation=4g -e HOME=/tmp") {
                    checkout scm
                    sh script: """
                        cd website
                        npm install
                        (npm run lint && npm run spellcheck) || { echo "
                        If there are spell check errors:
                        1) Suppressing False Positives: Edit website/.spelling to add suppressions. Instructions
                        are at the top of the file and explain how to suppress false positives either globally or
                        within a particular file.
                        2) Running Spell Check Locally: cd website && npm install && npm run spellcheck
                        For more information, refer to: https://www.npmjs.com/package/markdown-spellcheck
                        " && false; }
                    """
                }
            }
        }
    },
    "security vulnerabilities": {
        stage("security vulnerabilities") {
            lightweightNode {
                withM2Cache { cacheDir ->
                    docker.image(buildModuleTestImage("maven:3.6.3-jdk-8")).inside(
                        "--memory=8g --memory-reservation=4g -v ${cacheDir}/.m2:/root/.m2 -u root:root") {
                        checkout scm
                        withArtifactorySettings {
                            sh script: """
                                ${MVN} dependency-check:check || { echo "
                                The OWASP dependency check has found security vulnerabilities. Please use a newer version
                                of the dependency that does not have vulnerabilities. If the analysis has false positives,
                                they can be suppressed by adding entries to owasp-dependency-check-suppressions.xml (for more
                                information, see https://jeremylong.github.io/DependencyCheck/general/suppression.html).
                                " && false; }
                            """
                        }
                    }
                }
            }
        }
    },
    "(Compile=openjdk8, Run=openjdk8) batch index integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk8) batch index integration test", 8, ['TESTNG_GROUPS=-Dgroups=batch-index','JVM_RUNTIME=-Djvm.runtime=8'])
    },
    "(Compile=openjdk8, Run=openjdk8) input format integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk8) input format integration test", 8, ['TESTNG_GROUPS=-Dgroups=input-format','JVM_RUNTIME=-Djvm.runtime=8'])
    },
    "(Compile=openjdk8, Run=openjdk8) perfect rollup parallel batch index integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk8) perfect rollup parallel batch index integration test", 8, ['TESTNG_GROUPS=-Dgroups=perfect-rollup-parallel-batch-index','JVM_RUNTIME=-Djvm.runtime=8'])
    },
    "(Compile=openjdk8, Run=openjdk8) kafka index integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk8) kafka index integration test", 8, ['TESTNG_GROUPS=-Dgroups=kafka-index','JVM_RUNTIME=-Djvm.runtime=8'])
    },
    "(Compile=openjdk8, Run=openjdk8) kafka index integration test slow": {
        integrationTests("(Compile=openjdk8, Run=openjdk8) kafka index integration test slow", 8, ['TESTNG_GROUPS=-Dgroups=kafka-index-slow','JVM_RUNTIME=-Djvm.runtime=8'])
    },
    "(Compile=openjdk8, Run=openjdk8) transactional kafka index integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk8) transactional kafka index integration test", 8, ['TESTNG_GROUPS=-Dgroups=kafka-transactional-index','JVM_RUNTIME=-Djvm.runtime=8'])
    },
    "(Compile=openjdk8, Run=openjdk8) transactional kafka index integration test slow": {
        integrationTests("(Compile=openjdk8, Run=openjdk8) transactional kafka index integration test slow", 8, ['TESTNG_GROUPS=-Dgroups=kafka-transactional-index-slow','JVM_RUNTIME=-Djvm.runtime=8'])
    },
    "(Compile=openjdk8, Run=openjdk8) Kafka index integration test with various formats": {
        integrationTests("(Compile=openjdk8, Run=openjdk8) Kafka index integration test with various formats", 8, ['TESTNG_GROUPS=-Dgroups=kafka-data-format','JVM_RUNTIME=-Djvm.runtime=8'])
    },
    "(Compile=openjdk8, Run=openjdk8) query integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk8) query integration test", 8, ['TESTNG_GROUPS=-Dgroups=query','JVM_RUNTIME=-Djvm.runtime=8'])
    },
    "(Compile=openjdk8, Run=openjdk8) security integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk8) security integration test", 8, ['TESTNG_GROUPS=-Dgroups=security','JVM_RUNTIME=-Djvm.runtime=8'])
    },
    "(Compile=openjdk8, Run=openjdk8) realtime index integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk8) realtime index integration test", 8, ['TESTNG_GROUPS=-Dgroups=realtime-index','JVM_RUNTIME=-Djvm.runtime=8'])
    },
    "(Compile=openjdk8, Run=openjdk8) other integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk8) other integration test", 8, ['TESTNG_GROUPS=-DexcludedGroups=batch-index,perfect-rollup-parallel-batch-index,kafka-index,query,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage','JVM_RUNTIME=-Djvm.runtime=8'])
    },

    "(Compile=openjdk8, Run=openjdk11) batch index integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk11) batch index integration test", 8, ['TESTNG_GROUPS=-Dgroups=batch-index','JVM_RUNTIME=-Djvm.runtime=11'])
    },
    "(Compile=openjdk8, Run=openjdk11) input format integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk11) input format integration test", 8, ['TESTNG_GROUPS=-Dgroups=input-format','JVM_RUNTIME=-Djvm.runtime=11'])
    },
    "(Compile=openjdk8, Run=openjdk11) perfect rollup parallel batch index integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk11) perfect rollup parallel batch index integration test", 8, ['TESTNG_GROUPS=-Dgroups=perfect-rollup-parallel-batch-index','JVM_RUNTIME=-Djvm.runtime=11'])
    },
    "(Compile=openjdk8, Run=openjdk11) query integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk11) query integration test", 8, ['TESTNG_GROUPS=-Dgroups=query','JVM_RUNTIME=-Djvm.runtime=11'])
    },
    "(Compile=openjdk8, Run=openjdk11) security integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk11) security integration test", 8, ['TESTNG_GROUPS=-Dgroups=security','JVM_RUNTIME=-Djvm.runtime=11'])
    },
    "(Compile=openjdk8, Run=openjdk11) realtime index integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk11) realtime index integration test", 8, ['TESTNG_GROUPS=-Dgroups=realtime-index','JVM_RUNTIME=-Djvm.runtime=11'])
    },
    "(Compile=openjdk8, Run=openjdk11 other integration test": {
        integrationTests("(Compile=openjdk8, Run=openjdk11) other integration test", 8, ['TESTNG_GROUPS=-DexcludedGroups=batch-index,perfect-rollup-parallel-batch-index,kafka-index,query,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage','JVM_RUNTIME=-Djvm.runtime=11'])
    },

    "(Compile=openjdk8, Run=openjdk8) s3 deep storage test": {
        s3DeepStorageTests("(Compile=openjdk8, Run=openjdk8) s3 deep storage test", 8, ['JVM_RUNTIME=-Djvm.runtime=8'])
    },
    "(Compile=openjdk8, Run=openjdk8) kinesis deep storage test": {
        kinesisDeepStorageTests("(Compile=openjdk8, Run=openjdk8) kinesis deep storage test", 8, ['JVM_RUNTIME=-Djvm.runtime=8'])
    },
    "(Compile=openjdk8, Run=openjdk8) azure deep storage test": {
        azureDeepStorageTests("(Compile=openjdk8, Run=openjdk8) azure deep storage test", 8, ['JVM_RUNTIME=-Djvm.runtime=8'])
    },
    "(Compile=openjdk8, Run=openjdk8) hdfs deep storage test": {
        hdfsDeepStorageTests("(Compile=openjdk8, Run=openjdk8) hdfs deep storage test", 8, ['JVM_RUNTIME=-Djvm.runtime=8'])
    },

    "(Compile=openjdk8, Run=openjdk11) s3 deep storage test": {
        s3DeepStorageTests("(Compile=openjdk8, Run=openjdk11) s3 deep storage test", 8, ['JVM_RUNTIME=-Djvm.runtime=11'])
    },
    "(Compile=openjdk8, Run=openjdk11) azure deep storage test": {
        azureDeepStorageTests("(Compile=openjdk8, Run=openjdk11) azure deep storage test", 8, ['JVM_RUNTIME=-Djvm.runtime=11'])
    },
    "(Compile=openjdk8, Run=openjdk11) hdfs deep storage test": {
        hdfsDeepStorageTests("(Compile=openjdk8, Run=openjdk11) hdfs deep storage test", 8, ['JVM_RUNTIME=-Djvm.runtime=11'])
    }
}