Merge branch 'upstream-master' into add-asciidoctor

.github/workflows/build.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java_version: [8, 11]
+        java_version: [19]
 
     steps:
       - name: Environment

buildSrc/src/main/groovy/nextflow/gradle/plugins/SourcesMatcher.groovy

@@ -41,7 +41,7 @@ class SourcesMatcher {
         }
         matcher.collect { file ->
             def source = file.toString() - "$root.absolutePath/src/main/"
-            return source.split('\\.').dropRight(1).join().split(File.separator).drop(1).join('.')
+            return source.split('\\.').dropRight(1).join().split(File.separator).join('.')
         }
     }
 

gradle.properties

@@ -1 +1 @@
-version=0.0.2-rc1
+version=0.0.2

plugins/build.gradle

@@ -32,6 +32,21 @@ subprojects {
         mavenCentral()
     }
 
+    java {
+        toolchain {
+            languageVersion = JavaLanguageVersion.of(19)
+        }
+    }
+
+    compileJava {
+        options.release.set(11)
+    }
+
+    tasks.withType(GroovyCompile) {
+        sourceCompatibility = '11'
+        targetCompatibility = '11'
+    }
+
     tasks.withType(Jar) {
         duplicatesStrategy = DuplicatesStrategy.INCLUDE
     }

plugins/nf-nomad/build.gradle

@@ -95,5 +95,18 @@ dependencies {
 // use JUnit 5 platform
 test {
     useJUnitPlatform()
+    jvmArgs '--add-opens=java.base/java.lang=ALL-UNNAMED'
 }
 
+jar {
+    manifest {
+        attributes(
+                'Manifest-Version':'1.0',
+                'Plugin-Id': project.name,
+                'Plugin-Version': archiveVersion,
+                'Plugin-Class': "nextflow.nomad.NomadPlugin",
+                'Plugin-Provider': 'nextflow',
+                'Plugin-Requires': '>=23.10.0',
+        )
+    }
+}

plugins/nf-nomad/src/main/nextflow/nomad/NomadConfig.groovy

@@ -28,21 +28,33 @@ import groovy.util.logging.Slf4j
 @Slf4j
 @CompileStatic
 class NomadConfig {
+    final static protected API_VERSION = "v1"
+
+    final static public String VOLUME_DOCKER_TYPE = "docker"
+    final static public String VOLUME_CSI_TYPE = "csi"
+    final static public String VOLUME_HOST_TYPE = "host"
+
+    final static protected String[] VOLUME_TYPES = [
+            VOLUME_CSI_TYPE, VOLUME_DOCKER_TYPE, VOLUME_HOST_TYPE
+    ]
 
     final NomadClientOpts clientOpts
     final NomadJobOpts jobOpts
 
     NomadConfig(Map nomadConfigMap) {
-        clientOpts = new NomadClientOpts((nomadConfigMap.client ?: Collections.emptyMap()) as Map)
-        jobOpts = new NomadJobOpts((nomadConfigMap.jobs ?: Collections.emptyMap()) as Map)
+        clientOpts = new NomadClientOpts((nomadConfigMap?.client ?: Collections.emptyMap()) as Map)
+        jobOpts = new NomadJobOpts((nomadConfigMap?.jobs ?: Collections.emptyMap()) as Map)
     }
 
     class NomadClientOpts{
         final String address
         final String token
 
         NomadClientOpts(Map nomadClientOpts){
-            address = (nomadClientOpts.address?.toString() ?: "http://127.0.0.1:4646")+"/v1"
+            def tmp = (nomadClientOpts.address?.toString() ?: "http://127.0.0.1:4646")
+            if( !tmp.endsWith("/"))
+                tmp +="/"
+            this.address = tmp + API_VERSION
             token = nomadClientOpts.token ?: null
         }
     }
@@ -53,15 +65,67 @@ class NomadConfig {
         final String region
         final String namespace
         final String dockerVolume
+        final VolumeSpec volumeSpec
 
         NomadJobOpts(Map nomadJobOpts){
             deleteOnCompletion = nomadJobOpts.containsKey("deleteOnCompletion") ?
                     nomadJobOpts.deleteOnCompletion : false
-            datacenters = (nomadJobOpts.containsKey("datacenters") ?
-                    nomadJobOpts.datacenters.toString().split(",") : List.of("dc1")) as List<String>
+            if( nomadJobOpts.containsKey("datacenters") ) {
+                datacenters = ((nomadJobOpts.datacenters instanceof List<String> ?
+                        nomadJobOpts.datacenters : nomadJobOpts.datacenters.toString().split(","))
+                        as List<String>).findAll{it.size()}.unique()
+            }else{
+                datacenters = []
+            }
             region = nomadJobOpts.region ?: null
             namespace = nomadJobOpts.namespace ?: null
             dockerVolume = nomadJobOpts.dockerVolume ?: null
+            if( dockerVolume ){
+                log.info "dockerVolume config will be deprecated, use volume type:'docker' name:'name' instead"
+            }
+            if( nomadJobOpts.volume && nomadJobOpts.volume instanceof Closure){
+                this.volumeSpec = new VolumeSpec()
+                def closure = (nomadJobOpts.volume as Closure)
+                def clone = closure.rehydrate(this.volumeSpec, closure.owner, closure.thisObject)
+                clone.resolveStrategy = Closure.DELEGATE_FIRST
+                clone()
+                this.volumeSpec.validate()
+            }else{
+                volumeSpec = null
+            }
+        }
+    }
+
+    class VolumeSpec{
+
+        private String type
+        private String name
+
+        String getType() {
+            return type
+        }
+
+        String getName() {
+            return name
+        }
+
+        VolumeSpec type(String type){
+            this.type = type
+            this
+        }
+
+        VolumeSpec name(String name){
+            this.name = name
+            this
+        }
+
+        protected validate(){
+            if( !VOLUME_TYPES.contains(type) ) {
+                throw new IllegalArgumentException("Volume type $type is not supported")
+            }
+            if( !this.name ){
+                throw new IllegalArgumentException("Volume name is required")
+            }
         }
     }
 }

plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy

@@ -1,4 +1,5 @@
 /*
+ * Copyright 2024, Evaluacion y Desarrollo de Negocios, Spain
  * Copyright 2023, Stellenbosch University, South Africa
  * Copyright 2022, Center for Medical Genetics, Ghent
  *
@@ -28,6 +29,8 @@ import io.nomadproject.client.models.JobSummary
 import io.nomadproject.client.models.Task
 import io.nomadproject.client.models.TaskGroup
 import io.nomadproject.client.models.TaskGroupSummary
+import io.nomadproject.client.models.VolumeMount
+import io.nomadproject.client.models.VolumeRequest
 import nextflow.nomad.NomadConfig
 
 /**
@@ -80,6 +83,15 @@ class NomadService implements Closeable{
                 name: "group",
                 tasks: [ task ]
         )
+        if( config.jobOpts.volumeSpec){
+            taskGroup.volumes = [:]
+            taskGroup.volumes[config.jobOpts.volumeSpec.name]= new VolumeRequest(
+                    type: config.jobOpts.volumeSpec.type,
+                    source: config.jobOpts.volumeSpec.name,
+                    attachmentMode: "file-system",
+                    accessMode: "multi-node-multi-writer"
+            )
+        }
         return taskGroup
     }
 
@@ -105,13 +117,20 @@ class NomadService implements Closeable{
                     readonly : false
             ]
         }
+        if( config.jobOpts.volumeSpec){
+            String destinationDir = workingDir.split(File.separator).dropRight(2).join(File.separator)
+            task.volumeMounts = [ new VolumeMount(
+                    destination: destinationDir,
+                    volume: config.jobOpts.volumeSpec.name
+            )]
+        }
         task
     }
 
 
     String state(String jobId){
         JobSummary summary = jobsApi.getJobSummary(jobId, config.jobOpts.region, config.jobOpts.namespace, null, null, null, null, null, null, null)
-        TaskGroupSummary taskGroupSummary = summary.summary.values().first()
+        TaskGroupSummary taskGroupSummary = summary?.summary?.values()?.first()
         switch (taskGroupSummary){
             case {taskGroupSummary?.starting }:
                 return TaskGroupSummary.SERIALIZED_NAME_STARTING

plugins/nf-nomad/src/test/nextflow/nomad/NomadConfigSpec.groovy

@@ -0,0 +1,162 @@
+/*
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.nomad
+
+import spock.lang.Specification
+
+/**
+ * Unit test for Nomad Config
+ *
+ * @author : Jorge Aguilera <[email protected]>
+ */
+class NomadConfigSpec extends Specification {
+
+    void "should create a default config"() {
+        given:
+        def config = new NomadConfig()
+
+        expect:
+        config.jobOpts
+        config.clientOpts
+    }
+
+    void "should use localhost as default address"() {
+        given:
+        def config = new NomadConfig([:])
+
+        expect:
+        config.clientOpts.address == "http://127.0.0.1:4646/v1"
+    }
+
+    void "should use address if provided"() {
+        given:
+        def config = new NomadConfig([
+                client: [address: "http://nomad"]
+        ])
+
+        expect:
+        config.clientOpts.address == "http://nomad/v1"
+    }
+
+    void "should normalize address if provided"() {
+        given:
+        def config = new NomadConfig([
+                client: [address: "http://nomad/"]
+        ])
+
+        expect:
+        config.clientOpts.address == "http://nomad/v1"
+    }
+
+    void "should use token if provided"() {
+        given:
+        def config = new NomadConfig([
+                client: [token: "theToken"]
+        ])
+
+        expect:
+        config.clientOpts.token == "theToken"
+    }
+
+    void "should use an empty list if no datacenters is provided"() {
+        given:
+        def config = new NomadConfig([
+                jobs: [:]
+        ])
+
+        expect:
+        !config.jobOpts.datacenters.size()
+    }
+
+    void "should use datacenters #dc with size #size if provided"() {
+        given:
+        def config = new NomadConfig([
+                jobs: [datacenters: dc]
+        ])
+
+        expect:
+        config.jobOpts.datacenters.size() == size
+
+        where:
+        dc             | size
+        []             | 0
+        "dc1"          | 1
+        ['dc1']        | 1
+        "dc1,dc2"      | 2
+        ['dc1', 'dc2'] | 2
+        ['dc1', 'dc1'] | 1
+    }
+
+    void "should use region if provided"() {
+        given:
+        def config = new NomadConfig([
+                jobs: [region: "theRegion"]
+        ])
+
+        expect:
+        config.jobOpts.region == "theRegion"
+    }
+
+    void "should use namespace if provided"() {
+        given:
+        def config = new NomadConfig([
+                jobs: [namespace: "namespace"]
+        ])
+
+        expect:
+        config.jobOpts.namespace == "namespace"
+    }
+
+    void "should instantiate a volume spec if specified"() {
+        when:
+        def config = new NomadConfig([
+                jobs: [volume : { type "docker" name "test" }]
+        ])
+
+        then:
+        config.jobOpts.volumeSpec
+        config.jobOpts.volumeSpec.type == NomadConfig.VOLUME_DOCKER_TYPE
+        config.jobOpts.volumeSpec.name == "test"
+
+        when:
+        def config2 = new NomadConfig([
+                jobs: [volume : { type "csi" name "test" }]
+        ])
+
+        then:
+        config2.jobOpts.volumeSpec
+        config2.jobOpts.volumeSpec.type == NomadConfig.VOLUME_CSI_TYPE
+        config2.jobOpts.volumeSpec.name == "test"
+
+        when:
+        def config3 = new NomadConfig([
+                jobs: [volume  : { type "host" name "test" }]
+        ])
+
+        then:
+        config3.jobOpts.volumeSpec
+        config3.jobOpts.volumeSpec.type == NomadConfig.VOLUME_HOST_TYPE
+        config3.jobOpts.volumeSpec.name == "test"
+
+        when:
+        new NomadConfig([
+                jobs: [volume : { type "not-supported" name "test" }]
+        ])
+
+        then:
+        thrown(IllegalArgumentException)
+    }
+}