Iteration on Nomad/Variables <-> Nextflow/Secrets

buildSrc/src/main/groovy/nextflow/gradle/plugins/SourcesMatcher.groovy

@@ -22,7 +22,7 @@ class SourcesMatcher {
     }
 
     List<String> getProviders(){
-        return findSources(/class (\w+) implements (.+)Provider/)
+        return findSources(/implements SecretsProvider/)
     }
 
 

plugins/nf-nomad/src/main/nextflow/nomad/config/NomadJobOpts.groovy

@@ -40,14 +40,15 @@ class NomadJobOpts{
     List<String> datacenters
     String region
     String namespace
-    String secretsPath
     String dockerVolume
     JobVolume[] volumeSpec
     JobAffinity affinitySpec
     JobConstraint constraintSpec
 
     JobConstraints constraintsSpec
 
+    NomadSecretOpts secretOpts
+
     NomadJobOpts(Map nomadJobOpts, Map<String,String> env=null){
         assert nomadJobOpts!=null
 
@@ -77,8 +78,7 @@ class NomadJobOpts{
         this.affinitySpec = parseAffinity(nomadJobOpts)
         this.constraintSpec = parseConstraint(nomadJobOpts)
         this.constraintsSpec = parseConstraints(nomadJobOpts)
-
-        this.secretsPath = nomadJobOpts.secretsPath ?: sysEnv.get('NOMAD_SECRETS_PATH') ?: "secrets/nf-nomad"
+        this.secretOpts = parseSecrets(nomadJobOpts)
     }
 
     JobVolume[] parseVolumes(Map nomadJobOpts){
@@ -161,4 +161,14 @@ class NomadJobOpts{
             null
         }
     }
+
+    NomadSecretOpts parseSecrets(Map nomadJobOpts){
+        if (nomadJobOpts.secrets && nomadJobOpts.secrets instanceof Map) {
+            def secretOpts = new NomadSecretOpts(nomadJobOpts.secrets as Map)
+            secretOpts
+        }else{
+            null
+        }
+    }
+
 }

plugins/nf-nomad/src/main/nextflow/nomad/config/NomadSecretOpts.groovy

@@ -0,0 +1,13 @@
+package nextflow.nomad.config
+
+class NomadSecretOpts {
+
+    final Boolean enable
+    final String path
+
+    NomadSecretOpts(Map map){
+        this.enable = map.containsKey('enable') ? map.get('enable') as boolean : false
+        this.path = map.path ?: "secrets/nf-nomad"
+    }
+
+}

plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy

@@ -284,15 +284,17 @@ class NomadService implements Closeable{
     }
 
     protected Task secrets(TaskRun task, Task taskDef){
-        def secrets = task.processor?.config?.get(TaskDirectives.SECRETS)
-        if( secrets ){
-            Template template = new Template(envvars: true, destPath: "/secrets/nf-nomad")
-            String secretPath = config.jobOpts().secretsPath
-            String tmpl = secrets.collect{ String name->
-                "${name}={{ with nomadVar \"$secretPath/${name}\" }}{{ .${name} }}{{ end }}"
-            }.join('\n').stripIndent()
-            template.embeddedTmpl(tmpl)
-            taskDef.addTemplatesItem(template)
+        if( config.jobOpts()?.secretOpts?.enable) {
+            def secrets = task.processor?.config?.get(TaskDirectives.SECRETS)
+            if (secrets) {
+                Template template = new Template(envvars: true, destPath: "/secrets/nf-nomad")
+                String secretPath = config.jobOpts()?.secretOpts?.path
+                String tmpl = secrets.collect { String name ->
+                    "${name}={{ with nomadVar \"$secretPath/${name}\" }}{{ .${name} }}{{ end }}"
+                }.join('\n').stripIndent()
+                template.embeddedTmpl(tmpl)
+                taskDef.addTemplatesItem(template)
+            }
         }
         taskDef
     }
@@ -386,7 +388,7 @@ class NomadService implements Closeable{
     }
 
     String getVariableValue(String key){
-        getVariableValue(config.jobOpts().secretsPath, key)
+        getVariableValue(config.jobOpts().secretOpts?.path, key)
     }
 
     String getVariableValue(String path, String key){
@@ -398,7 +400,7 @@ class NomadService implements Closeable{
     }
 
     void setVariableValue(String key, String value){
-        setVariableValue(config.jobOpts().secretsPath, key, value)
+        setVariableValue(config.jobOpts().secretOpts?.path, key, value)
     }
 
     void setVariableValue(String path, String key, String value){
@@ -419,7 +421,7 @@ class NomadService implements Closeable{
     }
 
     void deleteVariable(String key){
-        deleteVariable(config.jobOpts().secretsPath, key)
+        deleteVariable(config.jobOpts().secretOpts?.path, key)
     }
 
     void deleteVariable(String path, String key){

plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretProvider.groovy

@@ -1,60 +1,80 @@
 package nextflow.nomad.secrets
 
 import groovy.util.logging.Slf4j
+import nextflow.Global
+import nextflow.nomad.config.NomadConfig
+import nextflow.nomad.executor.NomadService
 import nextflow.plugin.Priority
+import nextflow.secret.LocalSecretsProvider
 import nextflow.secret.Secret
+import nextflow.secret.SecretImpl
 import nextflow.secret.SecretsProvider
 
 @Slf4j
 @Priority(-100) // high priority
-class NomadSecretProvider implements SecretsProvider, Closeable{
+class NomadSecretProvider extends LocalSecretsProvider implements SecretsProvider {
 
-    @Override
-    void close() throws IOException {
-    }
+    NomadConfig config
 
     @Override
-    boolean activable() {
-        return true
+    LocalSecretsProvider load() {
+        return super.load()
     }
 
-    @Override
-    SecretsProvider load() {
-        this
+    protected boolean isEnabled(){
+        if( !config ){
+            config = new NomadConfig(Global.config?.nomad as Map ?: Map.of())
+        }
+        config?.jobOpts()?.secretOpts?.enable
     }
 
     @Override
     Secret getSecret(String name) {
-        log.debug("NomadSecretProvider can't get secret, use nomad cli or nextflow plugin nf-nomad:secrets")
-        null
+        if( !this.enabled ) {
+            return super.getSecret(name)
+        }
+        NomadService service = new NomadService(config)
+        String value = service.getVariableValue(name)
+        return new SecretImpl(name: name, value: value)
     }
 
     @Override
     String getSecretsEnv(List<String> secretNames) {
-        log.debug("NomadSecretProvider can't get secret, use nomad cli or nextflow plugin nf-nomad:secrets")
+        if( !this.enabled ) {
+            return super.getSecretsEnv(secretNames)
+        }
         null
     }
 
     @Override
     String getSecretsEnv() {
-        log.debug("NomadSecretProvider can't get secret, use nomad cli or nextflow plugin nf-nomad:secrets")
+        if( !this.enabled ) {
+            return super.getSecretsEnv()
+        }
         null
     }
 
     @Override
     void putSecret(String name, String value) {
-        throw new UnsupportedOperationException("NomadSecretProvider can't put secret, use nomad cli or nextflow plugin nf-nomad:secrets")
+        if( !this.enabled ) {
+            super.putSecret(name, value)
+        }
     }
 
     @Override
     void removeSecret(String name) {
-        throw new UnsupportedOperationException("NomadSecretProvider can't remove secret, use nomad cli or nextflow plugin nf-nomad:secrets")
+        if( !this.enabled ) {
+            super.removeSecret(name)
+        }
     }
 
     @Override
     Set<String> listSecretsNames() {
-        log.debug("NomadSecretProvider can't get secret, use nomad cli or nextflow plugin nf-nomad:secrets")
-        null
+        if( !this.enabled ) {
+            return super.listSecretsNames()
+        }
+        NomadService service = new NomadService(config)
+        service.variablesList as Set<String>
     }
 
 }

validation/secrets/main.nf

@@ -18,3 +18,6 @@ process sayHello {
 workflow {
     Channel.of('Bonjour', 'Ciao', 'Hello', 'Hola') | sayHello | view
 }
+workflow.onComplete {
+    println("The secret is: ${secrets.MY_ACCESS_KEY}")
+}

validation/secrets/nextflow.config

@@ -16,6 +16,10 @@ nomad {
         deleteOnCompletion = false
         volume = { type "host" name "scratchdir" }
         namespace = 'ns-qa'
+
+        secrets {
+            enable = true //if false then use LocalSecretsProvider implementation
+        }
     }
 
 }