Skip to content

Commit

Permalink
enable nomad secrets via nextflow config
Browse files Browse the repository at this point in the history
if false use Local implementation

Signed-off-by: Jorge Aguilera <[email protected]>
  • Loading branch information
jagedn committed Jul 27, 2024
1 parent 38721e9 commit b39b079
Show file tree
Hide file tree
Showing 7 changed files with 85 additions and 33 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ class SourcesMatcher {
}

List<String> getProviders(){
return findSources(/class (\w+) implements (.+)Provider/)
return findSources(/implements SecretsProvider/)
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,15 @@ class NomadJobOpts{
List<String> datacenters
String region
String namespace
String secretsPath
String dockerVolume
JobVolume[] volumeSpec
JobAffinity affinitySpec
JobConstraint constraintSpec

JobConstraints constraintsSpec

NomadSecretOpts secretOpts

NomadJobOpts(Map nomadJobOpts, Map<String,String> env=null){
assert nomadJobOpts!=null

Expand Down Expand Up @@ -77,8 +78,7 @@ class NomadJobOpts{
this.affinitySpec = parseAffinity(nomadJobOpts)
this.constraintSpec = parseConstraint(nomadJobOpts)
this.constraintsSpec = parseConstraints(nomadJobOpts)

this.secretsPath = nomadJobOpts.secretsPath ?: sysEnv.get('NOMAD_SECRETS_PATH') ?: "secrets/nf-nomad"
this.secretOpts = parseSecrets(nomadJobOpts)
}

JobVolume[] parseVolumes(Map nomadJobOpts){
Expand Down Expand Up @@ -161,4 +161,14 @@ class NomadJobOpts{
null
}
}

NomadSecretOpts parseSecrets(Map nomadJobOpts){
if (nomadJobOpts.secrets && nomadJobOpts.secrets instanceof Map) {
def secretOpts = new NomadSecretOpts(nomadJobOpts.secrets as Map)
secretOpts
}else{
null
}
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package nextflow.nomad.config

class NomadSecretOpts {

final Boolean enable
final String path

NomadSecretOpts(Map map){
this.enable = map.containsKey('enable') ? map.get('enable') as boolean : false
this.path = map.path ?: "secrets/nf-nomad"
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -284,15 +284,17 @@ class NomadService implements Closeable{
}

protected Task secrets(TaskRun task, Task taskDef){
def secrets = task.processor?.config?.get(TaskDirectives.SECRETS)
if( secrets ){
Template template = new Template(envvars: true, destPath: "/secrets/nf-nomad")
String secretPath = config.jobOpts().secretsPath
String tmpl = secrets.collect{ String name->
"${name}={{ with nomadVar \"$secretPath/${name}\" }}{{ .${name} }}{{ end }}"
}.join('\n').stripIndent()
template.embeddedTmpl(tmpl)
taskDef.addTemplatesItem(template)
if( config.jobOpts()?.secretOpts?.enable) {
def secrets = task.processor?.config?.get(TaskDirectives.SECRETS)
if (secrets) {
Template template = new Template(envvars: true, destPath: "/secrets/nf-nomad")
String secretPath = config.jobOpts()?.secretOpts?.path
String tmpl = secrets.collect { String name ->
"${name}={{ with nomadVar \"$secretPath/${name}\" }}{{ .${name} }}{{ end }}"
}.join('\n').stripIndent()
template.embeddedTmpl(tmpl)
taskDef.addTemplatesItem(template)
}
}
taskDef
}
Expand Down Expand Up @@ -386,7 +388,7 @@ class NomadService implements Closeable{
}

String getVariableValue(String key){
getVariableValue(config.jobOpts().secretsPath, key)
getVariableValue(config.jobOpts().secretOpts?.path, key)
}

String getVariableValue(String path, String key){
Expand All @@ -398,7 +400,7 @@ class NomadService implements Closeable{
}

void setVariableValue(String key, String value){
setVariableValue(config.jobOpts().secretsPath, key, value)
setVariableValue(config.jobOpts().secretOpts?.path, key, value)
}

void setVariableValue(String path, String key, String value){
Expand All @@ -419,7 +421,7 @@ class NomadService implements Closeable{
}

void deleteVariable(String key){
deleteVariable(config.jobOpts().secretsPath, key)
deleteVariable(config.jobOpts().secretOpts?.path, key)
}

void deleteVariable(String path, String key){
Expand Down
Original file line number Diff line number Diff line change
@@ -1,60 +1,80 @@
package nextflow.nomad.secrets

import groovy.util.logging.Slf4j
import nextflow.Global
import nextflow.nomad.config.NomadConfig
import nextflow.nomad.executor.NomadService
import nextflow.plugin.Priority
import nextflow.secret.LocalSecretsProvider
import nextflow.secret.Secret
import nextflow.secret.SecretImpl
import nextflow.secret.SecretsProvider

@Slf4j
@Priority(-100) // high priority
class NomadSecretProvider implements SecretsProvider, Closeable{
class NomadSecretProvider extends LocalSecretsProvider implements SecretsProvider {

@Override
void close() throws IOException {
}
NomadConfig config

@Override
boolean activable() {
return true
LocalSecretsProvider load() {
return super.load()
}

@Override
SecretsProvider load() {
this
protected boolean isEnabled(){
if( !config ){
config = new NomadConfig(Global.config?.nomad as Map ?: Map.of())
}
config?.jobOpts()?.secretOpts?.enable
}

@Override
Secret getSecret(String name) {
log.debug("NomadSecretProvider can't get secret, use nomad cli or nextflow plugin nf-nomad:secrets")
null
if( !this.enabled ) {
return super.getSecret(name)
}
NomadService service = new NomadService(config)
String value = service.getVariableValue(name)
return new SecretImpl(name: name, value: value)
}

@Override
String getSecretsEnv(List<String> secretNames) {
log.debug("NomadSecretProvider can't get secret, use nomad cli or nextflow plugin nf-nomad:secrets")
if( !this.enabled ) {
return super.getSecretsEnv(secretNames)
}
null
}

@Override
String getSecretsEnv() {
log.debug("NomadSecretProvider can't get secret, use nomad cli or nextflow plugin nf-nomad:secrets")
if( !this.enabled ) {
return super.getSecretsEnv()
}
null
}

@Override
void putSecret(String name, String value) {
throw new UnsupportedOperationException("NomadSecretProvider can't put secret, use nomad cli or nextflow plugin nf-nomad:secrets")
if( !this.enabled ) {
super.putSecret(name, value)
}
}

@Override
void removeSecret(String name) {
throw new UnsupportedOperationException("NomadSecretProvider can't remove secret, use nomad cli or nextflow plugin nf-nomad:secrets")
if( !this.enabled ) {
super.removeSecret(name)
}
}

@Override
Set<String> listSecretsNames() {
log.debug("NomadSecretProvider can't get secret, use nomad cli or nextflow plugin nf-nomad:secrets")
null
if( !this.enabled ) {
return super.listSecretsNames()
}
NomadService service = new NomadService(config)
service.variablesList as Set<String>
}

}
3 changes: 3 additions & 0 deletions validation/secrets/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,6 @@ process sayHello {
workflow {
Channel.of('Bonjour', 'Ciao', 'Hello', 'Hola') | sayHello | view
}
workflow.onComplete {
println("The secret is: ${secrets.MY_ACCESS_KEY}")
}
4 changes: 4 additions & 0 deletions validation/secrets/nextflow.config
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ nomad {
deleteOnCompletion = false
volume = { type "host" name "scratchdir" }
namespace = 'ns-qa'

secrets {
enable = true //if false then use LocalSecretsProvider implementation
}
}

}
Expand Down

0 comments on commit b39b079

Please sign in to comment.