build.gradle

//Note: this section 'buildscript` is only for the dependencies of the buildscript itself.
// See the second 'repositories' section below for the actual dependencies of GATK itself
buildscript {
    repositories {
        mavenCentral()
     }
}

plugins {
    id "java"           // set up default java compile and test tasks
    id "application"    // provides installDist
    id 'maven-publish'
    id 'signing'
    id "jacoco"
    id "de.undercouch.download" version "4.1.2" //used for downloading GSA lib
    id "com.github.johnrengelman.shadow" version "7.1.1"    //used to build the shadow and sparkJars
    id "com.github.ben-manes.versions" version "0.12.0" //used for identifying dependencies that need updating
    id 'com.palantir.git-version' version '0.5.1' //version helper
}


import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
import de.undercouch.gradle.tasks.download.Download

import javax.tools.ToolProvider
import java.time.format.DateTimeFormatter
import java.time.ZonedDateTime

mainClassName = "org.broadinstitute.hellbender.Main"

//Note: the test suite must use the same defaults. If you change system properties in this list you must also update the one in the test task
applicationDefaultJvmArgs = ["-Dsamjdk.use_async_io_read_samtools=false","-Dsamjdk.use_async_io_write_samtools=true", "-Dsamjdk.use_async_io_write_tribble=false", "-Dsamjdk.compression_level=2"]

//Delete the windows script - we never test on Windows so let's not pretend it works
startScripts {
    doLast {
        delete windowsScript
    }
}

task downloadGsaLibFile(type: Download) {
    src 'http://cran.r-project.org/src/contrib/gsalib_2.1.tar.gz'
    dest "src/main/resources/org/broadinstitute/hellbender/utils/R/gsalib.tar.gz"
    overwrite false
}


repositories {
    mavenCentral()

    maven {
        url "https://broadinstitute.jfrog.io/broadinstitute/libs-snapshot/" //for htsjdk snapshots
    }

    maven {
        url "https://oss.sonatype.org/content/repositories/snapshots" //for disq snapshots
    }

    mavenLocal()
}

final htsjdkVersion = System.getProperty('htsjdk.version','2.24.1')
final picardVersion = System.getProperty('picard.version','2.25.4')
final barclayVersion = System.getProperty('barclay.version','4.0.2')
final sparkVersion = System.getProperty('spark.version', '2.4.5')
final scalaVersion = System.getProperty('scala.version', '2.11')
final hadoopVersion = System.getProperty('hadoop.version', '3.2.1')
final disqVersion = System.getProperty('disq.version','0.3.6')
final genomicsdbVersion = System.getProperty('genomicsdb.version','1.4.3')
final bigQueryVersion = System.getProperty('bigQuery.version', '1.117.1')
final guavaVersion = System.getProperty('guava.version', '27.1-jre')
final log4j2Version = System.getProperty('log4j2Version', '2.17.1')
final testNGVersion = '7.0.0'

// Using the shaded version to avoid conflicts between its protobuf dependency
// and that of Hadoop/Spark (either the one we reference explicitly, or the one
// provided by dataproc).
final googleCloudNioDependency = 'com.google.cloud:google-cloud-nio:0.117.0-alpha:shaded'

final baseJarName = 'gatk'
final secondaryBaseJarName = 'hellbender'
final docBuildDir = "$buildDir/docs"
final pythonPackageArchiveName = 'gatkPythonPackageArchive.zip'
final gatkCondaTemplate = "gatkcondaenv.yml.template"
final gatkCondaYML = "gatkcondaenv.yml"
final largeResourcesFolder = "src/main/resources/large"
final buildPrerequisitesMessage = "See https://github.com/broadinstitute/gatk#building for information on how to build GATK."

// Returns true if any files in the target folder are git-lfs stub files.
def checkForLFSStubFiles(targetFolder) {
    final lfsStubFileHeader = "version https://git-lfs.github.com/spec/v1"  // first line of a git-lfs stub file
    def readBytesFromFile = { largeFile, n ->
        final byte[] bytes = new byte[n]
        largeFile.withInputStream { stream -> stream.read(bytes, 0, bytes.length) }
        return bytes
    }
    def targetFiles = fileTree(dir: targetFolder)
    return targetFiles.any() { f ->
        final byte[] actualBytes = readBytesFromFile(f, lfsStubFileHeader.length());
        return new String(actualBytes, "UTF-8") == lfsStubFileHeader
    }
}

// if any of the large resources are lfs stub files, download them
def resolveLargeResourceStubFiles(largeResourcesFolder, buildPrerequisitesMessage) {
    def execGitLFSCommand = { gitLFSExecCommand ->
        println "Executing: $gitLFSExecCommand"
        try {
            def retCode = gitLFSExecCommand.execute().waitFor()
            if (retCode.intValue() != 0) {
                throw new GradleException("Execution of \"$gitLFSExecCommand\" failed with exit code: $retCode. " +
                        " git-lfs is required to build GATK but may not be installed. $buildPrerequisitesMessage");
            }
            return retCode
        } catch (IOException e) {
            throw new GradleException(
                    "An IOException occurred while attempting to execute the command $gitLFSExecCommand."
                    + " git-lfs is required to build GATK but may not be installed. $buildPrerequisitesMessage", e)
        }
    }

    // check for stub files, try to pull once if there are any, then check again
    if (checkForLFSStubFiles(largeResourcesFolder)) {
        final gitLFSPullLargeResources = "git lfs pull --include $largeResourcesFolder"
        execGitLFSCommand(gitLFSPullLargeResources)
        if (checkForLFSStubFiles(largeResourcesFolder)) {
            throw new GradleException("$largeResourcesFolder contains one or more git-lfs stub files."
                    + " The resource files in $largeResourcesFolder must be downloaded by running the git-lfs"
                    + " command \"$gitLFSPullLargeResources\". $buildPrerequisitesMessage")
        }
    }
}

// Check that we're in a folder which git recognizes as a git repository.
// This works for either a standard git clone or one created with `git worktree add`
def looksLikeWereInAGitRepository(){
    file(".git").isDirectory() || (file(".git").exists() && file(".git").text.startsWith("gitdir"))
}

// Ensure that we have a clone of the git repository, and resolve any required git-lfs
// resource files that are needed to run the build but are still lfs stub files.
def ensureBuildPrerequisites(largeResourcesFolder, buildPrerequisitesMessage, skipGitCheck) {
    if (!JavaVersion.current().isJava8Compatible()) {
        throw new GradleException(
                "Java 8 or later is required to build GATK, but ${JavaVersion.current()} was found. "
                        + "$buildPrerequisitesMessage")
    }
    // Make sure we can get a ToolProvider class loader (for Java 8). If not we may have just a JRE.
    if (JavaVersion.current().isJava8() && ToolProvider.getSystemToolClassLoader() == null) {
        throw new GradleException(
                "The ClassLoader obtained from the Java ToolProvider is null. "
                + "A full Java 8 or 11 JDK must be installed, check that you are not using a JRE. $buildPrerequisitesMessage")
    }
    if (!JavaVersion.current().isJava8() && !JavaVersion.current().isJava11()) {
        println("Warning: using Java ${JavaVersion.current()} but only Java 8 and Java 11 have been tested.")
    }
    if (!skipGitCheck && !looksLikeWereInAGitRepository() ) {
        throw new GradleException("This doesn't appear to be a git folder. " +
                "The GATK Github repository must be cloned using \"git clone\" to run the build. " +
                "\n$buildPrerequisitesMessage")
    }
    // Large runtime resource files must be present at build time to be compiled into the jar, so
    // try to resolve them to real files if any of them are stubs.
    resolveLargeResourceStubFiles(largeResourcesFolder, buildPrerequisitesMessage)
}

final isRelease = Boolean.getBoolean("release")
final versionOverridden = System.getProperty("versionOverride") != null

ensureBuildPrerequisites(largeResourcesFolder, buildPrerequisitesMessage, versionOverridden)
version = (versionOverridden ? System.getProperty("versionOverride") : gitVersion().replaceAll(".dirty", "")) + (isRelease ?  "" : "-SNAPSHOT")

if (versionOverridden) {
    println "Version number overridden as " + version
}

configurations.all {
    resolutionStrategy {
        // the snapshot folder contains a dev version of guava, we don't want to use that.
        force 'com.google.guava:guava:' + guavaVersion
        // force the htsjdk version so we don't get a different one transitively
        force 'com.github.samtools:htsjdk:' + htsjdkVersion
        force 'com.google.protobuf:protobuf-java:3.8.0'
        // force testng dependency so we don't pick up a different version via GenomicsDB
        force 'org.testng:testng:' + testNGVersion
        force 'org.broadinstitute:barclay:' + barclayVersion
        force 'com.twitter:chill_2.11:0.8.1'

        // make sure we don't pick up an incorrect version of the GATK variant of the google-nio library
        // via Picard, etc.
        force googleCloudNioDependency
    }
    all*.exclude group: 'org.slf4j', module: 'slf4j-jdk14' //exclude this to prevent slf4j complaining about to many slf4j bindings
    all*.exclude group: 'com.google.guava', module: 'guava-jdk5'
    all*.exclude group: 'junit', module: 'junit'
}

tasks.withType(JavaCompile) {
  options.compilerArgs = ['-proc:none', '-Xlint:all', '-Werror', '-Xdiags:verbose']
  options.encoding = 'UTF-8'
}

sourceSets {
    testUtils
}

// Dependency change for including MLLib
configurations {
    testUtilsImplementation.extendsFrom implementation
    testUtilsRuntimeClasspath.extendsFrom runtimeClasspath

    testImplementation.extendsFrom testUtilsImplementation
    testRuntimeClasspath.extendsFrom testUtilsRuntimeClasspath

    implementation.exclude module: 'jul-to-slf4j'
    implementation.exclude module: 'javax.servlet'
    implementation.exclude module: 'servlet-api'
    implementation.exclude group: 'com.esotericsoftware.kryo'

    externalSourceConfiguration {
        // External sources we need for doc and tab completion generation tasks (i.e., Picard sources)
        transitive false
    }

    sparkConfiguration {
        extendsFrom runtimeClasspath
        // exclude Hadoop and Spark dependencies, since they are provided when running with Spark
        // (ref: http://unethicalblogger.com/2015/07/15/gradle-goodness-excluding-depends-from-shadow.html)
        exclude group: 'org.apache.hadoop'
        exclude module: 'spark-core_2.11'
        exclude group: 'org.slf4j'
        exclude module: 'jul-to-slf4j'
        exclude module: 'javax.servlet'
        exclude module: 'servlet-api'
        exclude group: 'com.esotericsoftware.kryo'
        exclude module: 'spark-mllib_2.11'
        exclude group: 'org.scala-lang'
        exclude module: 'kryo'
    }
}

// Get the jdk files we need to run javaDoc. We need to use these during compile, testCompile,
// test execution, and gatkDoc generation, but we don't want them as part of the runtime
// classpath and we don't want to redistribute them in the uber jar.
final javadocJDKFiles = ToolProvider.getSystemToolClassLoader() == null ? files([]) : files(((URLClassLoader) ToolProvider.getSystemToolClassLoader()).getURLs())

dependencies {
    // javadoc utilities; compile/test only to prevent redistribution of sdk jars
    compileOnly(javadocJDKFiles)
    testImplementation(javadocJDKFiles)

    implementation 'org.broadinstitute:barclay:' + barclayVersion
    // Library for configuration:
    implementation 'org.aeonbits.owner:owner:1.0.9'

    implementation 'com.github.broadinstitute:picard:' + picardVersion
    externalSourceConfiguration 'com.github.broadinstitute:picard:' + picardVersion + ':sources'
    implementation ('org.genomicsdb:genomicsdb:' + genomicsdbVersion)  {
        exclude module: 'log4j-api'
        exclude module: 'log4j-core'
        exclude module: 'spark-core_2.12'
        exclude module: 'spark-sql_2.12'
        exclude module: 'htsjdk'
        exclude module: 'protobuf-java'
    }
    implementation 'com.opencsv:opencsv:3.4'
    implementation 'com.google.guava:guava:' + guavaVersion
    implementation 'com.github.samtools:htsjdk:'+ htsjdkVersion
    implementation(googleCloudNioDependency) {
        transitive = false
    }

    implementation 'com.google.cloud:google-cloud-bigquery:' + bigQueryVersion
    implementation 'com.google.cloud:google-cloud-bigquerystorage:1.5.1'

    implementation "gov.nist.math.jama:gov.nist.math.jama:1.1.1"

    // this comes built-in when running on Google Dataproc, but the library
    // allows us to read from GCS also when testing locally (or on non-Dataproc clusters,
    // should we want to)
    implementation 'com.google.cloud.bigdataoss:gcs-connector:1.9.4-hadoop3'

    implementation 'org.apache.logging.log4j:log4j-api:' + log4j2Version
    implementation 'org.apache.logging.log4j:log4j-core:' + log4j2Version
    // include the apache commons-logging bridge that matches the log4j version we use so
    // messages that originate with dependencies that use commons-logging (such as jexl)
    // are routed to log4j
    implementation 'org.apache.logging.log4j:log4j-jcl:' + log4j2Version

    implementation 'org.apache.commons:commons-lang3:3.5'
    implementation 'org.apache.commons:commons-math3:3.5'
    implementation 'org.apache.commons:commons-collections4:4.1'
    implementation 'org.apache.commons:commons-vfs2:2.0'
    implementation 'org.apache.commons:commons-configuration2:2.4'
    implementation 'org.apache.httpcomponents:httpclient:4.5.12'
    implementation 'commons-beanutils:commons-beanutils:1.9.3'
    implementation 'commons-io:commons-io:2.5'
    implementation 'org.reflections:reflections:0.9.10'

    implementation 'it.unimi.dsi:fastutil:7.0.6'

    implementation 'org.broadinstitute:hdf5-java-bindings:1.1.0-hdf5_2.11.0'
    implementation 'org.broadinstitute:gatk-native-bindings:1.0.0'

    implementation 'org.ojalgo:ojalgo:44.0.0'
    implementation ('org.ojalgo:ojalgo-commons-math3:1.0.0') {
        exclude group: 'org.apache.commons'
    }
    implementation ('org.apache.spark:spark-mllib_' + scalaVersion + ':' + sparkVersion) {
        // JUL is used by Google Dataflow as the backend logger, so exclude jul-to-slf4j to avoid a loop
        exclude module: 'jul-to-slf4j'
        exclude module: 'javax.servlet'
        exclude module: 'servlet-api'
    }
    implementation 'com.thoughtworks.paranamer:paranamer:2.8'

    implementation 'org.bdgenomics.bdg-formats:bdg-formats:0.5.0'
    implementation('org.bdgenomics.adam:adam-core-spark2_' + scalaVersion + ':0.28.0') {
        exclude group: 'org.slf4j'
        exclude group: 'org.apache.hadoop'
        exclude group: 'org.scala-lang'
        exclude module: 'kryo'
        exclude module: 'hadoop-bam'
    }

    implementation 'org.jgrapht:jgrapht-core:0.9.1'

    implementation('org.disq-bio:disq:' + disqVersion)
    implementation('org.apache.hadoop:hadoop-client:' + hadoopVersion) // should be a 'provided' dependency
    implementation('com.github.jsr203hadoop:jsr203hadoop:1.0.3')

    implementation('de.javakaffee:kryo-serializers:0.41') {
        exclude module: 'kryo' // use Spark's version
    }

    // Dependency change for including MLLib
    implementation('org.objenesis:objenesis:1.2')
    testImplementation('org.objenesis:objenesis:2.1')

    // Comment the next lines to disable native code proxies in Spark MLLib
    implementation('com.github.fommil.netlib:netlib-native_ref-osx-x86_64:1.1:natives')
    implementation('com.github.fommil.netlib:netlib-native_ref-linux-x86_64:1.1:natives')
    implementation('com.github.fommil.netlib:netlib-native_system-linux-x86_64:1.1:natives')
    implementation('com.github.fommil.netlib:netlib-native_system-osx-x86_64:1.1:natives')

    // Dependency change for including MLLib
    implementation('com.esotericsoftware:kryo:3.0.3'){
        exclude group: 'com.esotericsoftware', module: 'reflectasm'
        exclude group: 'org.ow2.asm', module: 'asm'
    }

    // Dependency change for including MLLib
    implementation('com.esotericsoftware:reflectasm:1.10.0:shaded') {
        transitive = false
    }

    implementation('com.intel.gkl:gkl:0.8.8') {
        exclude module: 'htsjdk'
    }

    implementation 'org.broadinstitute:gatk-bwamem-jni:1.0.4'
    implementation 'org.broadinstitute:gatk-fermilite-jni:1.2.0'

    implementation 'org.broadinstitute:http-nio:0.1.0-rc1'

    // Required for COSMIC Funcotator data source:
    implementation 'org.xerial:sqlite-jdbc:3.36.0.3'
    
    // Required for SV Discovery machine learning
    implementation group: 'biz.k11i', name: 'xgboost-predictor', version: '0.3.0'

    // natural sort
    implementation('net.grey-panther:natural-comparator:1.1')
    implementation('com.fasterxml.jackson.module:jackson-module-scala_' + scalaVersion + ':2.9.8')

    testUtilsImplementation sourceSets.main.output
    testUtilsImplementation 'org.testng:testng:' + testNGVersion
    testUtilsImplementation 'org.apache.hadoop:hadoop-minicluster:' + hadoopVersion

    testImplementation sourceSets.testUtils.output

    testImplementation "org.mockito:mockito-core:2.28.2"
    testImplementation "com.google.jimfs:jimfs:1.1"
}

//add gatk launcher script to the jar as a resource
processResources {
    from("gatk")
}

processTestResources {
    //Don't waste time packaging unnecessary test data into the test resources:
    include "org/broadinstitute/hellbender/utils/config/*"
    //Required for IOUtils resource tests
    include "org/broadinstitute/hellbender/utils/io/*"
}

sourceCompatibility = 1.8
targetCompatibility = 1.8

def createSymlinks(archivePath, symlinkLocation) {
    exec {
        commandLine 'ln', '-fs', archivePath, symlinkLocation
        ignoreExitValue = false
    }
}

// Suffix is what will be added to the symlink
def createGatkSymlinks(destinationDir, archivePath, suffix, baseJarName, secondaryBaseJarName) {
    def finalSuffix = (suffix == "") ? "" : ("-" + suffix)

    def symlinkLocation = destinationDir.getAsFile().get().toString() + "/" + baseJarName + finalSuffix + ".jar"
    def symlinkLocation2 = destinationDir.getAsFile().get().toString() + "/" + secondaryBaseJarName + finalSuffix + ".jar"

    createSymlinks(archivePath.getAbsolutePath(), symlinkLocation)
    createSymlinks(archivePath.getAbsolutePath(), symlinkLocation2)
}

logger.info("build for version:" + version)
group = 'org.broadinstitute'

tasks.withType(Jar) {
    manifest {
        attributes 'Implementation-Title': 'The Genome Analysis Toolkit (GATK)',
                'Implementation-Version': archiveVersion,
                'Toolkit-Short-Name' : 'GATK',
                'Main-Class': project.mainClassName,
                'Picard-Version': picardVersion,
                'htsjdk-Version': htsjdkVersion,
                'Spark-Version': sparkVersion,
                'Multi-Release': 'true'
    }
}

wrapper {
    gradleVersion = '7.3.2'
}

tasks.withType(ShadowJar) {
    from(project.sourceSets.main.output)
    archiveBaseName = project.name + '-package'
    mergeServiceFiles()
    relocate 'com.google.common', 'org.broadinstitute.hellbender.relocated.com.google.common'
    zip64 true
    exclude 'log4j.properties' // from adam jar as it clashes with hellbender's log4j2.xml
    exclude '**/*.SF' // these are Manifest signature files and
    exclude '**/*.RSA' // keys which may accidentally be imported from other signed projects and then fail at runtime

    // Suggested by the akka devs to make sure that we do not get the spark configuration error.
    // http://doc.akka.io/docs/akka/snapshot/general/configuration.html#When_using_JarJar__OneJar__Assembly_or_any_jar-bundler
    transform(com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer) {
        resource = 'reference.conf'
    }
}

apply from: "testsettings.gradle"

shadowJar {
    configurations = [project.configurations.runtimeClasspath]
    archiveClassifier = 'local'
    mergeServiceFiles('reference.conf')
    doLast {
        // Create a symlink to the newly created jar.  The name will be gatk.jar and
        //  it will be at the same level as the newly created jar.  (overwriting symlink, if it exists)
        // Please note that this will cause failures in Windows, which does not support symlinks.
        createGatkSymlinks(destinationDirectory, archivePath, "", baseJarName, secondaryBaseJarName)
    }
}

task localJar{ dependsOn shadowJar }

task sparkJar(type: ShadowJar) {
    group = "Shadow"
    description = "Create a combined jar of project and runtime dependencies that excludes provided spark dependencies"
    configurations = [project.configurations.sparkConfiguration]
    archiveClassifier = 'spark'
    doLast {
        // Create a symlink to the newly created jar.  The name will be gatk.jar and
        //  it will be at the same level as the newly created jar.  (overwriting symlink, if it exists)
        // Please note that this will cause failures in Windows, which does not support symlinks.
        createGatkSymlinks(destinationDirectory, archivePath, archiveClassifier, baseJarName, secondaryBaseJarName)
    }
}

// A jar that only contains the test classes and resources (to be extracted for testing)
task shadowTestClassJar(type: ShadowJar){
    group = "Shadow"
    from sourceSets.test.output
    description = "Create a jar that packages the compiled test classes"
    archiveClassifier = "test"
}

// A minimal jar that only contains the extra dependencies needed for running the tests
task shadowTestJar(type: ShadowJar){
    group = "Shadow"
    description = " A minimal jar that only contains the extra dependencies needed for running the tests that arent packaged in the main shadow jar"
    from {
        (project.configurations.testRuntimeClasspath - project.configurations.runtimeClasspath ).collect {
            it.isDirectory() ? it : it.getName().endsWith(".jar") ? zipTree(it) : it
        }
    }
    archiveClassifier = "testDependencies"
}

task collectBundleIntoDir(type: Copy) {
    dependsOn shadowJar, sparkJar, 'condaEnvironmentDefinition', 'gatkTabComplete', 'gatkDoc'

    doFirst {
        assert file("gatk").exists()
        assert file("README.md").exists()
        assert file("$docBuildDir/tabCompletion/gatk-completion.sh").exists()
        assert file("src/main/resources/org/broadinstitute/hellbender/utils/config/GATKConfig.properties").exists()
    }

    from(shadowJar.archivePath)
    from(sparkJar.archivePath)
    from("gatk")
    from("README.md")
    from("$docBuildDir/tabCompletion/gatk-completion.sh")
    from("$docBuildDir/gatkDoc", { into("gatkdoc") })

    from("src/main/resources/org/broadinstitute/hellbender/utils/config/GATKConfig.properties") {
        rename 'GATKConfig.properties', 'GATKConfig.EXAMPLE.properties'
    }

    from("$buildDir/$pythonPackageArchiveName")
    from("$buildDir/$gatkCondaYML")
    from("scripts/sv", { into("scripts/sv") })
    from("scripts/cnv_wdl/", { into("scripts/cnv_wdl") })
    from("scripts/mutect2_wdl/", { into("scripts/mutect2_wdl") })
    from("scripts/dataproc-cluster-ui", { into("scripts/")})
    into "$buildDir/bundle-files-collected"
}

task bundle(type: Zip) {
    dependsOn collectBundleIntoDir

    archiveBaseName = project.name + "-" + project.version
    destinationDirectory = file("$buildDir")
    archiveFileName = archiveBaseName.get() + ".zip"

    from("$buildDir/bundle-files-collected")
    into(archiveBaseName)

    doLast {
        logger.lifecycle("Created GATK distribution in ${destinationDir}/${archiveName}")
    }
}

jacocoTestReport {
    dependsOn test

    group = "Reporting"
    description = "Generate Jacoco coverage reports after running tests."
    getAdditionalSourceDirs().from(sourceSets.main.allJava.srcDirs)

    reports {
        xml.required = true
        html.required = true
    }
}

task condaStandardEnvironmentDefinition(type: Copy) {
    from "scripts"
    into buildDir
    include gatkCondaTemplate
    rename { file -> gatkCondaYML }
    expand(["condaEnvName":"gatk",
            "condaEnvDescription" : "Conda environment for GATK Python Tools"])
    doLast {
        logger.lifecycle("Created standard Conda environment yml file: $gatkCondaYML")
    }
}


// Create GATK conda environment yml file from the conda enc template
task condaEnvironmentDefinition() {
    dependsOn 'pythonPackageArchive', 'condaStandardEnvironmentDefinition'
}

// Create the Python package archive file
task pythonPackageArchive(type: Zip) {
    inputs.dir "src/main/python/org/broadinstitute/hellbender/"
    outputs.file pythonPackageArchiveName
    doFirst {
        assert file("src/main/python/org/broadinstitute/hellbender/").exists()
    }

    destinationDirectory = file("${buildDir}")
    archiveFileName = pythonPackageArchiveName
    from("src/main/python/org/broadinstitute/hellbender/")
    into("/")

    doLast {
        logger.lifecycle("Created GATK Python package archive in ${destinationDir}/${archiveName}")
    }
}

// Creates a standard, local, GATK conda environment, for use by developers during iterative
// development. Assumes conda or miniconda is already installed.
//
// NOTE: This CREATES a local conda environment; but does not *activate* it. The environment must
// be activated manually in the shell from which GATK will be run.
//
task localDevCondaEnv(type: Exec) {
    dependsOn 'condaEnvironmentDefinition'
    inputs.file("$buildDir/$pythonPackageArchiveName")
    workingDir "$buildDir"
    commandLine "conda", "env", "create", "--force", "-f", gatkCondaYML
}

task javadocJar(type: Jar, dependsOn: javadoc) {
    archiveClassifier = 'javadoc'
    from "$docBuildDir/javadoc"
}

task sourcesJar(type: Jar) {
    from sourceSets.main.allSource
    archiveClassifier = 'sources'
}

task testUtilsJar(type: Jar){
    archiveBaseName = "$project.name-test-utils"
    from sourceSets.testUtils.output
}

tasks.withType(Javadoc) {
    // do this for all javadoc tasks, including gatkDoc
    options.addStringOption('Xdoclint:none')
    options.addStringOption('encoding', 'UTF-8')
}

javadoc {
    // This is a hack to disable the java 8 default javadoc lint until we fix the html formatting
    // We only want to do this for the javadoc task, not gatkDoc
    options.addStringOption('Xdoclint:none', '-quiet')
    source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
    include '**/*.java'
}


task testUtilsJavadoc(type: Javadoc) {
    // This is a hack to disable the java 8 default javadoc lint until we fix the html formatting
    // We only want to do this for the javadoc task, not gatkDoc
    options.addStringOption('Xdoclint:none', '-quiet')
    source = sourceSets.testUtils.allJava
    classpath = sourceSets.testUtils.runtimeClasspath
    destinationDir = file("$docBuildDir/testUtilsJavadoc")
    include '**/*.java'
}

task testUtilsJavadocJar(type: Jar, dependsOn: testUtilsJavadoc){
    archiveBaseName = "$project.name-test-utils"
    archiveClassifier = 'javadoc'
    from "$docBuildDir/testUtilsJavadoc"
}

task testUtilsSourcesJar(type: Jar){
    archiveBaseName = "$project.name-test-utils"
    archiveClassifier = 'sources'
    from sourceSets.testUtils.allSource
}

// Generate GATK Online Doc
task gatkDoc(type: Javadoc, dependsOn: classes) {
    final File gatkDocDir = new File("$docBuildDir/gatkdoc")
    doFirst {
        // make sure the output folder exists or we can create it
        if (!gatkDocDir.exists() && !gatkDocDir.mkdirs()) {
            throw new GradleException(String.format("Failure creating folder (%s) for GATK doc output in task (%s)",
                    gatkDocDir.getAbsolutePath(),
                    it.name));
        }
        copy {
            from('src/main/resources/org/broadinstitute/hellbender/utils/helpTemplates')
            include 'gatkDoc.css'
            into gatkDocDir
        }
    }
    // Include the Picard source jar, which contains various .R, .sh, .css, .html, .xml and .MF files and
    // other resources, but we only want the files that javadoc can handle, so just take the .java files.
    source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
    include '**/*.java'

    // The gatkDoc process instantiates any documented feature classes, so to run it we need the entire
    // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives.
    classpath = sourceSets.main.runtimeClasspath + javadocJDKFiles
    options.docletpath = classpath.asType(List)
    options.doclet = "org.broadinstitute.hellbender.utils.help.GATKHelpDoclet"

    //gradle 6.x+ defaults to setting this true which breaks the barclay doclet
    options.noTimestamp(false)

    outputs.dir(gatkDocDir)
    options.destinationDirectory(gatkDocDir)

    options.addStringOption("settings-dir", "src/main/resources/org/broadinstitute/hellbender/utils/helpTemplates");
    if (project.hasProperty('phpDoc')) {
        // use -PphpDoc to generate .php file extensions, otherwise rely on default of .html
        final String phpExtension = "php"
        options.addStringOption("output-file-extension", phpExtension)
        options.addStringOption("index-file-extension", phpExtension)
    }
    options.addStringOption("absolute-version", getVersion())
    options.addStringOption("build-timestamp", ZonedDateTime.now().format(DateTimeFormatter.RFC_1123_DATE_TIME))
}

// Generate GATK Bash Tab Completion File
task gatkTabComplete(type: Javadoc, dependsOn: classes) {
    final File tabCompletionDir = new File("$docBuildDir/tabCompletion")
    doFirst {
        // make sure the output folder exists or we can create it
        if (!tabCompletionDir.exists() && !tabCompletionDir.mkdirs()) {
            throw new GradleException(String.format("Failure creating folder (%s) for GATK tab completion output in task (%s)",
                    tabCompletionDir.getAbsolutePath(),
                    it.name));
        }
    }
    // Include the Picard source jar, which contains various .R, .sh, .css, .html, .xml and .MF files and
    // other resources, but we only want the files that javadoc can handle, so just take the .java files.
    source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
    include '**/*.java'

    // The gatkDoc process instantiates any documented feature classes, so to run it we need the entire
    // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives, and Picard.
    classpath = sourceSets.main.runtimeClasspath + javadocJDKFiles

    options.docletpath = classpath.asType(List)
    options.doclet = "org.broadinstitute.barclay.help.BashTabCompletionDoclet"

    //gradle 6.x+ defaults to setting this true which breaks the barclay doclet
    options.noTimestamp(false)

    outputs.dir(tabCompletionDir)
    options.destinationDirectory(tabCompletionDir)

    // This is a hack to work around a gross Gradle bug:
    options.addStringOption('use-default-templates', '-use-default-templates')

    options.addStringOption("output-file-extension", "sh")
    options.addStringOption("index-file-extension", "sh")
    options.addStringOption("absolute-version", getVersion())
    options.addStringOption("build-timestamp", ZonedDateTime.now().format(DateTimeFormatter.RFC_1123_DATE_TIME))

    options.addStringOption("caller-script-name", "gatk")

    options.addStringOption("caller-pre-legal-args", "--help --list --dry-run --java-options")
    options.addStringOption("caller-pre-arg-val-types", "null null null String")
    options.addStringOption("caller-pre-mutex-args", "--help;list,dry-run,java-options --list;help,dry-run,java-options")
    options.addStringOption("caller-pre-alias-args", "--help;-h")
    options.addStringOption("caller-pre-arg-min-occurs", "0 0 0 0")
    options.addStringOption("caller-pre-arg-max-occurs", "1 1 1 1")

    options.addStringOption("caller-post-legal-args", "--spark-runner --spark-master --cluster --dry-run --java-options --conf --driver-memory --driver-cores --executor-memory --executor-cores --num-executors")
    options.addStringOption("caller-post-arg-val-types", "String String String null String file int int int int int")
    options.addStringOption("caller-post-mutex-args", "")
    options.addStringOption("caller-post-alias-args", "")
    options.addStringOption("caller-post-arg-min-occurs", "0 0 0 0 0 0 0 0 0 0")
    options.addStringOption("caller-post-arg-max-occurs", "1 1 1 1 1 1 1 1 1 1")
}

def getWDLInputJSONTestFileNameFromWDLName(File wdlName) {
    String fileWithoutExt = wdlName.name.take(wdlName.name.lastIndexOf('.'))
    return new File (wdlName.getParentFile(), fileWithoutExt + "Inputs.json").getAbsolutePath()
}

// Generate GATK Tool WDL
task gatkWDLGen(type: Javadoc, dependsOn: classes) {
    final File gatkWDLDir = new File("$docBuildDir/wdlGen")
    outputs.dir(gatkWDLDir)
    doFirst {
        // make sure the output folder exists or we can create it
        if (!gatkWDLDir.exists() && !gatkWDLDir.mkdirs()) {
            throw new GradleException(String.format("Failure creating folder (%s) for GATK WDL output in task (%s)",
                    gatkWDLDir.getAbsolutePath(),
                    it.name));
        }
        copy {
            from('src/main/resources/org/broadinstitute/hellbender/utils/wdlTemplates/common.html')
            into gatkWDLDir
        }
    }
    source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
    include '**/*.java'

    // The gatkWDLGen process instantiates any documented feature classes, so to run it we need the entire
    // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives.
    classpath = sourceSets.main.runtimeClasspath + javadocJDKFiles
    options.docletpath = classpath.asType(List)
    options.doclet = "org.broadinstitute.hellbender.utils.help.GATKWDLDoclet"

    //gradle 6.x+ defaults to setting this true which breaks the barclay doclet
    options.noTimestamp(false)

    outputs.dir(gatkWDLDir)
    options.destinationDirectory(gatkWDLDir)

    options.addStringOption("settings-dir", "src/main/resources/org/broadinstitute/hellbender/utils/wdlTemplates");
    options.addStringOption("output-file-extension", "wdl")
    options.addStringOption("index-file-extension", "html")

    options.addStringOption("absolute-version", getVersion())
    options.addStringOption("build-timestamp", ZonedDateTime.now().format(DateTimeFormatter.RFC_1123_DATE_TIME))

    // the wdl doclet will populate the test JSON input files with the name of a dummy
    // file in this location, in order to satisfy cromwell's attempts to localize inputs and outputs
    options.addStringOption("build-dir", System.getenv("TRAVIS_BUILD_DIR") ?: new File(".").getAbsolutePath())
}

task gatkWDLGenValidation(dependsOn: [gatkWDLGen, shadowJar]) {
    doFirst {
        // running this task requires a local cromwell installation, with environment variables CROMWELL_JAR,
        // WOMTOOL_JAR set to the jar locations
        if (System.getenv('CROMWELL_JAR') == null || System.getenv('WOMTOOL_JAR') == null) {
            throw new GradleException("Running this task requires the CROMWELL_JAR and WOMTOOL_JAR environment variables to be set")
        }
    }

    def execWDLValidation = { validateWDL ->
        println "Executing: $validateWDL"
        try {
            def retCode = validateWDL.execute().waitFor()
            if (retCode.intValue() != 0) {
                throw new GradleException("Execution of \"$validateWDL\" failed with exit code: $retCode.")
            }
            return retCode
        } catch (IOException e) {
            throw new GradleException("An IOException occurred while attempting to execute the command $validateWDL.")
        }
    }

    doLast {
        // first, run the womtool validator
        final File wdlGenFolder = new File("$docBuildDir/wdlGen")
        def wdlFiles = fileTree(dir: wdlGenFolder).filter {
            f -> !f.getAbsolutePath().endsWith(".html") && !f.getAbsolutePath().endsWith(".json")
        }
        final womtoolLocation = System.getenv('WOMTOOL_JAR')
        wdlFiles.any() { wdlFile ->
            final validateWDLCommand = "java -jar $womtoolLocation validate $wdlFile"
            execWDLValidation(validateWDLCommand)
        }

        // now execute the *AllArgs test wdls using cromwell
        wdlFiles = fileTree(dir: wdlGenFolder).filter {
            f -> f.getAbsolutePath().endsWith("AllArgsTest.wdl")
        }

        // the test JSON input file is populated by the WDL gen process with the name of this dummy file
        // to satisfy cromwell's attempt to de/localize input/output files
        def buildDir = System.getenv("TRAVIS_BUILD_DIR") ?: new File(".").getAbsolutePath()
        final dummyWDLTestFileName = "$buildDir/dummyWDLTestFile"
        final File dummyWDLTestFile = file(dummyWDLTestFileName)
        final cromwellLocation = System.getenv('CROMWELL_JAR')
        try {
            wdlFiles.any() { wdlFile ->
                final testInputJSON = getWDLInputJSONTestFileNameFromWDLName(wdlFile)
                final runWDLCommand = "java -jar $cromwellLocation run --inputs $testInputJSON $wdlFile"
                execWDLValidation("touch $dummyWDLTestFileName")
                execWDLValidation(runWDLCommand)
            }
        } finally {
            // delete the dummy test file and the 'cromwell-executions' directory left behind by cromwell
            dummyWDLTestFile.delete()
            file("$buildDir/cromwell-executions").deleteDir()
            file("$buildDir/cromwell-workflow-logs").deleteDir()
        }
    }

}


/**
 *This specifies what artifacts will be built and uploaded when performing a maven upload.
 */
artifacts {
    archives javadocJar
    archives sourcesJar
    archives testUtilsJar
    archives testUtilsJavadocJar
    archives testUtilsSourcesJar

}
//remove zip and tar added by the application plugin
configurations.archives.artifacts.removeAll {it.file =~ '.zip$'}
configurations.archives.artifacts.removeAll {it.file =~ '.tar$'}

/**
 * Sign non-snapshot releases with our secret key.  This should never need to be invoked directly.
 */
signing {
    required { isRelease && gradle.taskGraph.hasTask("publish") }
    sign publishing.publications
}

def basePomConfiguration = {
    packaging = 'jar'
    description = 'Development on GATK 4'
    url = 'http://github.com/broadinstitute/gatk'

    scm {
        url = 'scm:git@github.com:broadinstitute/gatk.git'
        connection = 'scm:git@github.com:broadinstitute/gatk.git'
        developerConnection = 'scm:git@github.com:broadinstitute/gatk.git'
    }

    developers {
        developer {
            id = 'gatkdev'
            name = 'GATK Development Team'
            email = 'gatk-dev-public@broadinstitute.org'
        }
    }

    licenses {
        license {
            name = 'Apache 2.0'
            url = 'https://github.com/broadinstitute/gatk/blob/master/LICENSE.TXT'
            distribution = 'repo'
        }
    }
}


//remove the shadow jar from the published component
components.java.withVariantsFromConfiguration(project.configurations.shadowRuntimeElements) {
    skip()
}

publishing {
    publications {
        gatk(MavenPublication) {
            from components.java
            artifactId = "gatk"
            pom basePomConfiguration
            pom.name = "GATK4"

            artifact sourcesJar
            artifact javadocJar
        }

        testUtils(MavenPublication) {
            artifactId = "gatk-test-utils"
            pom basePomConfiguration
            pom.name = "GATK4 Test Utilities"

            artifact testUtilsJar
            artifact testUtilsSourcesJar
            artifact testUtilsJavadocJar

        }
    }

    repositories {
        maven {
            name = isRelease ? "SonaType" : "Artifactory"
            url =  isRelease ? "https://oss.sonatype.org/service/local/staging/deploy/maven2/" : "https://broadinstitute.jfrog.io/broadinstitute/libs-snapshot-local/"
            credentials {
                username = isRelease ? project.findProperty("sonatypeUsername") : System.env.ARTIFACTORY_USERNAME
                password = isRelease ? project.findProperty("sonatypePassword") : System.env.ARTIFACTORY_PASSWORD
            }
        }
    }
}

publish {
    doFirst {
        println "Attempting to upload version:$version"
    }
}

task installSpark{ dependsOn sparkJar }
task installAll{  dependsOn installSpark, installDist }

installDist.dependsOn downloadGsaLibFile

defaultTasks 'bundle'