From e85940e1cfdc7e5679f2198f6030e5dc7b1213dc Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Mon, 29 Jul 2024 09:59:08 +0200 Subject: [PATCH] Iteration on Nomad/Variables <-> Nextflow/Secrets (#75) * first iteration in secrets implementation Signed-off-by: Jorge Aguilera * tweak the start-nomad script for different platforms [ci skip] * nomad secrets, add get set and list commands Signed-off-by: Jorge Aguilera * delete nomad with stop command [ci skip] * tweak the sun-nomadlab config to accommodate variables [ci skip] * update the sun-nomadlab config for config level secret vars [ci skip] * enable nomad secrets via nextflow config if false use Local implementation Signed-off-by: Jorge Aguilera * add test Signed-off-by: Jorge Aguilera * fix small bug Signed-off-by: Jorge Aguilera * improve local testing env [ci skip] * test with sun-nomadlab and localsecretstore [ci skip] * rename enable -> enabled to comply with standard * use the functional config for sun-nomadlab [ci skip] * update authors [ci skip] --------- Signed-off-by: Jorge Aguilera Co-authored-by: Jorge Aguilera --- .../gradle/plugins/GenerateIdxTask.groovy | 1 + .../gradle/plugins/SourcesMatcher.groovy | 5 + .../main/nextflow/nomad/NomadPlugin.groovy | 28 +++- .../nextflow/nomad/config/NomadJobOpts.groovy | 13 ++ .../nomad/config/NomadSecretOpts.groovy | 13 ++ .../nomad/executor/NomadService.groovy | 74 ++++++++++- .../nomad/executor/TaskDirectives.groovy | 5 +- .../nomad/secrets/NomadSecretCmd.groovy | 79 ++++++++++++ .../nomad/secrets/NomadSecretProvider.groovy | 85 +++++++++++++ .../src/resources/META-INF/MANIFEST.MF | 2 +- .../executor/NomadSecretServiceSpec.groovy | 120 ++++++++++++++++++ settings.gradle | 2 - validation/secrets/main.nf | 23 ++++ validation/secrets/nextflow.config | 26 ++++ validation/start-nomad.sh | 17 ++- validation/stop-nomad.sh | 3 +- validation/sun-nomadlab/nextflow.config | 20 ++- 17 files changed, 497 insertions(+), 19 deletions(-) create mode 100644 plugins/nf-nomad/src/main/nextflow/nomad/config/NomadSecretOpts.groovy create mode 100644 plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretCmd.groovy create mode 100644 plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretProvider.groovy create mode 100644 plugins/nf-nomad/src/test/nextflow/nomad/executor/NomadSecretServiceSpec.groovy create mode 100644 validation/secrets/main.nf create mode 100644 validation/secrets/nextflow.config diff --git a/buildSrc/src/main/groovy/nextflow/gradle/plugins/GenerateIdxTask.groovy b/buildSrc/src/main/groovy/nextflow/gradle/plugins/GenerateIdxTask.groovy index 4b8708f..289f475 100644 --- a/buildSrc/src/main/groovy/nextflow/gradle/plugins/GenerateIdxTask.groovy +++ b/buildSrc/src/main/groovy/nextflow/gradle/plugins/GenerateIdxTask.groovy @@ -34,6 +34,7 @@ abstract class GenerateIdxTask extends DefaultTask{ def matcher = new SourcesMatcher(project) def extensionsClassName = matcher.pluginExtensions + extensionsClassName += matcher.providers def traceClassName = matcher.traceObservers def all = extensionsClassName+traceClassName output.text = all.join('\n') diff --git a/buildSrc/src/main/groovy/nextflow/gradle/plugins/SourcesMatcher.groovy b/buildSrc/src/main/groovy/nextflow/gradle/plugins/SourcesMatcher.groovy index 1865b93..1beb1cc 100644 --- a/buildSrc/src/main/groovy/nextflow/gradle/plugins/SourcesMatcher.groovy +++ b/buildSrc/src/main/groovy/nextflow/gradle/plugins/SourcesMatcher.groovy @@ -21,6 +21,11 @@ class SourcesMatcher { findSources(/class (\w+) extends Executor implements ExtensionPoint/) } + List getProviders(){ + return findSources(/implements SecretsProvider/) + } + + List getTraceObservers(){ return findSources(/class (\w+) implements TraceObserverFactory/) } diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/NomadPlugin.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/NomadPlugin.groovy index 5772093..9c6d243 100644 --- a/plugins/nf-nomad/src/main/nextflow/nomad/NomadPlugin.groovy +++ b/plugins/nf-nomad/src/main/nextflow/nomad/NomadPlugin.groovy @@ -18,26 +18,50 @@ package nextflow.nomad import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.cli.PluginAbstractExec +import nextflow.nomad.secrets.NomadSecretCmd import nextflow.nomad.executor.TaskDirectives import nextflow.plugin.BasePlugin import nextflow.script.ProcessConfig +import nextflow.secret.SecretsLoader import org.pf4j.PluginWrapper /** * Nextflow plugin for Nomad executor * * @author Abhinav Sharma - * @author : matthdsm + * @author Jorge Aguilera */ @CompileStatic -class NomadPlugin extends BasePlugin { +@Slf4j +class NomadPlugin extends BasePlugin implements PluginAbstractExec{ NomadPlugin(PluginWrapper wrapper) { super(wrapper) addCustomDirectives() + SecretsLoader.instance.reset() } private static void addCustomDirectives() { ProcessConfig.DIRECTIVES.addAll(TaskDirectives.ALL) } + + @Override + List getCommands() { + return ['secrets'] + } + + @Override + int exec(String cmd, List args) { + return switch (cmd){ + case 'secrets'-> secrets(args.first(), args.drop(1)) + default -> -1 + } + } + + int secrets(String action, Listargs){ + NomadSecretCmd nomadSecretCmd = new NomadSecretCmd() + nomadSecretCmd.runCommand( session.config , action, args) + } } diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/config/NomadJobOpts.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/config/NomadJobOpts.groovy index a31dd3f..78ce744 100644 --- a/plugins/nf-nomad/src/main/nextflow/nomad/config/NomadJobOpts.groovy +++ b/plugins/nf-nomad/src/main/nextflow/nomad/config/NomadJobOpts.groovy @@ -47,6 +47,8 @@ class NomadJobOpts{ JobConstraints constraintsSpec + NomadSecretOpts secretOpts + NomadJobOpts(Map nomadJobOpts, Map env=null){ assert nomadJobOpts!=null @@ -76,6 +78,7 @@ class NomadJobOpts{ this.affinitySpec = parseAffinity(nomadJobOpts) this.constraintSpec = parseConstraint(nomadJobOpts) this.constraintsSpec = parseConstraints(nomadJobOpts) + this.secretOpts = parseSecrets(nomadJobOpts) } JobVolume[] parseVolumes(Map nomadJobOpts){ @@ -158,4 +161,14 @@ class NomadJobOpts{ null } } + + NomadSecretOpts parseSecrets(Map nomadJobOpts){ + if (nomadJobOpts.secrets && nomadJobOpts.secrets instanceof Map) { + def secretOpts = new NomadSecretOpts(nomadJobOpts.secrets as Map) + secretOpts + }else{ + null + } + } + } \ No newline at end of file diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/config/NomadSecretOpts.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/config/NomadSecretOpts.groovy new file mode 100644 index 0000000..ee6749e --- /dev/null +++ b/plugins/nf-nomad/src/main/nextflow/nomad/config/NomadSecretOpts.groovy @@ -0,0 +1,13 @@ +package nextflow.nomad.config + +class NomadSecretOpts { + + final Boolean enabled + final String path + + NomadSecretOpts(Map map){ + this.enabled = map.containsKey('enabled') ? map.get('enabled') as boolean : false + this.path = map.path ?: "secrets/nf-nomad" + } + +} diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy index 21cc80c..080e172 100644 --- a/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy +++ b/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy @@ -20,7 +20,9 @@ package nextflow.nomad.executor import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.nomadproject.client.ApiClient +import io.nomadproject.client.ApiException import io.nomadproject.client.api.JobsApi +import io.nomadproject.client.api.VariablesApi import io.nomadproject.client.model.* import nextflow.nomad.models.ConstraintsBuilder import nextflow.nomad.models.JobConstraints @@ -43,8 +45,9 @@ import java.nio.file.Path class NomadService implements Closeable{ NomadConfig config - + ApiClient apiClient JobsApi jobsApi + VariablesApi variablesApi NomadService(NomadConfig config) { this.config = config @@ -54,7 +57,7 @@ class NomadService implements Closeable{ final READ_TIMEOUT_MILLISECONDS = 60000 final WRITE_TIMEOUT_MILLISECONDS = 60000 - ApiClient apiClient = new ApiClient( connectTimeout: CONNECTION_TIMEOUT_MILLISECONDS, readTimeout: READ_TIMEOUT_MILLISECONDS, writeTimeout: WRITE_TIMEOUT_MILLISECONDS) + apiClient = new ApiClient( connectTimeout: CONNECTION_TIMEOUT_MILLISECONDS, readTimeout: READ_TIMEOUT_MILLISECONDS, writeTimeout: WRITE_TIMEOUT_MILLISECONDS) apiClient.basePath = config.clientOpts().address log.debug "[NOMAD] Client Address: ${config.clientOpts().address}" @@ -63,6 +66,7 @@ class NomadService implements Closeable{ apiClient.apiKey = config.clientOpts().token } this.jobsApi = new JobsApi(apiClient) + this.variablesApi = new VariablesApi(apiClient) } protected Resources getResources(TaskRun task) { @@ -110,6 +114,9 @@ class NomadService implements Closeable{ try { JobRegisterResponse jobRegisterResponse = jobsApi.registerJob(jobRegisterRequest, config.jobOpts().region, config.jobOpts().namespace, null, null) jobRegisterResponse.evalID + } catch( ApiException apiException){ + log.debug("[NOMAD] Failed to submit ${job.name} -- Cause: ${apiException.responseBody ?: apiException}", apiException) + throw new ProcessSubmitException("[NOMAD] Failed to submit ${job.name} -- Cause: ${apiException.responseBody ?: apiException}", apiException) } catch (Throwable e) { log.debug("[NOMAD] Failed to submit ${job.name} -- Cause: ${e.message ?: e}", e) throw new ProcessSubmitException("[NOMAD] Failed to submit ${job.name} -- Cause: ${e.message ?: e}", e) @@ -186,7 +193,7 @@ class NomadService implements Closeable{ affinity(task, taskDef) constraint(task, taskDef) constraints(task, taskDef) - + secrets(task, taskDef) return taskDef } @@ -276,7 +283,21 @@ class NomadService implements Closeable{ taskDef } - + protected Task secrets(TaskRun task, Task taskDef){ + if( config.jobOpts()?.secretOpts?.enabled) { + def secrets = task.processor?.config?.get(TaskDirectives.SECRETS) + if (secrets) { + Template template = new Template(envvars: true, destPath: "/secrets/nf-nomad") + String secretPath = config.jobOpts()?.secretOpts?.path + String tmpl = secrets.collect { String name -> + "${name}={{ with nomadVar \"$secretPath/${name}\" }}{{ .${name} }}{{ end }}" + }.join('\n').stripIndent() + template.embeddedTmpl(tmpl) + taskDef.addTemplatesItem(template) + } + } + taskDef + } protected Job assignDatacenters(TaskRun task, Job job){ def datacenters = task.processor?.config?.get(TaskDirectives.DATACENTERS) @@ -365,4 +386,49 @@ class NomadService implements Closeable{ throw new ProcessSubmitException("[NOMAD] Failed to get alloactions ${jobId} -- Cause: ${e.message ?: e}", e) } } + + String getVariableValue(String key){ + getVariableValue(config.jobOpts().secretOpts?.path, key) + } + + String getVariableValue(String path, String key){ + var variable = variablesApi.getVariableQuery("$path/$key", + config.jobOpts().region, + config.jobOpts().namespace, + null, null, null, null, null, null, null) + variable?.items?.find{ it.key == key }?.value + } + + void setVariableValue(String key, String value){ + setVariableValue(config.jobOpts().secretOpts?.path, key, value) + } + + void setVariableValue(String path, String key, String value){ + var content = Map.of(key,value) + var variable = new Variable(path: path, items: content) + variablesApi.postVariable("$path/$key", variable, + config.jobOpts().region, + config.jobOpts().namespace, + null, null, null) + } + + List getVariablesList(){ + var listRequest = variablesApi.getVariablesListRequest( + config.jobOpts().region, + config.jobOpts().namespace, + null, null, null, null, null, null, null) + listRequest.collect{ it.path} + } + + void deleteVariable(String key){ + deleteVariable(config.jobOpts().secretOpts?.path, key) + } + + void deleteVariable(String path, String key){ + var variable = new Variable( items: Map.of(key, "")) + variablesApi.deleteVariable("$path/$key", variable, + config.jobOpts().region, + config.jobOpts().namespace, + null, null, null) + } } diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/executor/TaskDirectives.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/executor/TaskDirectives.groovy index edc1722..e72c381 100644 --- a/plugins/nf-nomad/src/main/nextflow/nomad/executor/TaskDirectives.groovy +++ b/plugins/nf-nomad/src/main/nextflow/nomad/executor/TaskDirectives.groovy @@ -6,8 +6,11 @@ class TaskDirectives { public static final String CONSTRAINTS = "constraints" + public static final String SECRETS = "secret" + public static final List ALL = [ DATACENTERS, - CONSTRAINTS + CONSTRAINTS, + SECRETS ] } diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretCmd.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretCmd.groovy new file mode 100644 index 0000000..0ae2ba9 --- /dev/null +++ b/plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretCmd.groovy @@ -0,0 +1,79 @@ +package nextflow.nomad.secrets + +import groovy.util.logging.Slf4j +import nextflow.exception.AbortOperationException +import nextflow.nomad.config.NomadConfig +import nextflow.nomad.executor.NomadService +import nextflow.plugin.Priority +import nextflow.secret.Secret +import nextflow.secret.SecretImpl +import nextflow.secret.SecretsProvider + +@Slf4j +class NomadSecretCmd { + + protected NomadService service + protected NomadConfig nomadConfig + + int runCommand(Map config, String action, List args){ + nomadConfig = new NomadConfig((config.nomad ?: Collections.emptyMap()) as Map) + service = new NomadService(nomadConfig) + return switch (action){ + case 'get' ->execGetSecretNames(args.removeAt(0).toString()) + case 'set' ->execSetSecretNames(args.removeAt(0).toString(),args.removeAt(0).toString()) + case 'list'->execListSecretsNames() + case 'delete'->execDeleteSecretNames(args.removeAt(0).toString()) + default -> -1 + } + } + + int execListSecretsNames(){ + def list = listSecretsNames() + println list.join('\n') + return 0 + } + + int execGetSecretNames(String name){ + if(!name){ + throw new AbortOperationException("Wrong number of arguments") + } + def secret = getSecret(name) + println secret + return 0 + } + + int execSetSecretNames(String name, String value){ + if(!name){ + throw new AbortOperationException("Wrong number of arguments") + } + setSecret(name, value) + return 0 + } + + int execDeleteSecretNames(String name){ + if(!name){ + throw new AbortOperationException("Wrong number of arguments") + } + deleteSecret(name) + return 0 + } + + String getSecret(String name) { + String value = service.getVariableValue(name) + if( !value ) + throw new AbortOperationException("Missing secret name") + value + } + + Set listSecretsNames() { + service.variablesList + } + + void setSecret(String name, String value) { + service.setVariableValue(name, value) + } + + void deleteSecret(String name){ + service.deleteVariable(name) + } +} diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretProvider.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretProvider.groovy new file mode 100644 index 0000000..0375ec2 --- /dev/null +++ b/plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretProvider.groovy @@ -0,0 +1,85 @@ +package nextflow.nomad.secrets + +import groovy.util.logging.Slf4j +import nextflow.Global +import nextflow.nomad.config.NomadConfig +import nextflow.nomad.executor.NomadService +import nextflow.plugin.Priority +import nextflow.secret.LocalSecretsProvider +import nextflow.secret.Secret +import nextflow.secret.SecretImpl +import nextflow.secret.SecretsProvider + +/** + * Custom Nextflow CLI commands for nf-nomad + * + * @author Jorge Aguilera + */ +@Slf4j +@Priority(-100) // high priority +class NomadSecretProvider extends LocalSecretsProvider implements SecretsProvider { + + NomadConfig config + + @Override + LocalSecretsProvider load() { + return super.load() + } + + protected boolean isEnabled(){ + if( !config ){ + config = new NomadConfig(Global.config?.nomad as Map ?: Map.of()) + } + config?.jobOpts()?.secretOpts?.enabled + } + + @Override + Secret getSecret(String name) { + if( !this.enabled ) { + return super.getSecret(name) + } + NomadService service = new NomadService(config) + String value = service.getVariableValue(name) + return new SecretImpl(name: name, value: value) + } + + @Override + String getSecretsEnv(List secretNames) { + if( !this.enabled ) { + return super.getSecretsEnv(secretNames) + } + null + } + + @Override + String getSecretsEnv() { + if( !this.enabled ) { + return super.getSecretsEnv() + } + null + } + + @Override + void putSecret(String name, String value) { + if( !this.enabled ) { + super.putSecret(name, value) + } + } + + @Override + void removeSecret(String name) { + if( !this.enabled ) { + super.removeSecret(name) + } + } + + @Override + Set listSecretsNames() { + if( !this.enabled ) { + return super.listSecretsNames() + } + NomadService service = new NomadService(config) + service.variablesList as Set + } + +} diff --git a/plugins/nf-nomad/src/resources/META-INF/MANIFEST.MF b/plugins/nf-nomad/src/resources/META-INF/MANIFEST.MF index 6fe64c3..da26157 100644 --- a/plugins/nf-nomad/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-nomad/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Id: nf-nomad -Plugin-Version: 0.1.0 +Plugin-Version: 0.1.2 Plugin-Class: nextflow.nomad.NomadPlugin Plugin-Provider: nextflow Plugin-Requires: >=24.01.0-edge diff --git a/plugins/nf-nomad/src/test/nextflow/nomad/executor/NomadSecretServiceSpec.groovy b/plugins/nf-nomad/src/test/nextflow/nomad/executor/NomadSecretServiceSpec.groovy new file mode 100644 index 0000000..96ea68c --- /dev/null +++ b/plugins/nf-nomad/src/test/nextflow/nomad/executor/NomadSecretServiceSpec.groovy @@ -0,0 +1,120 @@ +/* + * Copyright 2023-, Stellenbosch University, South Africa + * Copyright 2024, Evaluacion y Desarrollo de Negocios, Spain + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nextflow.nomad.executor + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import nextflow.executor.Executor +import nextflow.nomad.config.NomadConfig +import nextflow.processor.TaskBean +import nextflow.processor.TaskConfig +import nextflow.processor.TaskProcessor +import nextflow.processor.TaskRun +import nextflow.script.ProcessConfig +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Unit test for Nomad Service for Secrets requests + * + * Validate requests using a Mock WebServer + * + * @author : Jorge Aguilera + */ +class NomadSecretServiceSpec extends Specification{ + + MockWebServer mockWebServer + + def setup() { + mockWebServer = new MockWebServer() + mockWebServer.start() + } + + def cleanup() { + mockWebServer.shutdown() + } + + void "should request a variable"(){ + given: + def config = new NomadConfig( + client:[ + address : "http://${mockWebServer.hostName}:${mockWebServer.port}" + ], + jobs: [ + secrets:[ + enable: true, + path: 'test' + ] + ] + ) + def service = new NomadService(config) + + when: + mockWebServer.enqueue(new MockResponse() + .addHeader("Content-Type", "application/json")); + + service.getVariableValue("MySecret") + def recordedRequest = mockWebServer.takeRequest(); + + then: + recordedRequest.method == "GET" + recordedRequest.path == "/v1/var/test%2FMySecret" + + when: + mockWebServer.enqueue(new MockResponse() + .addHeader("Content-Type", "application/json")); + + service.getVariableValue("another", "MySecret") + def recordedRequest2 = mockWebServer.takeRequest(); + + then: + recordedRequest2.method == "GET" + recordedRequest2.path == "/v1/var/another%2FMySecret" + } + + void "should set a variable"(){ + given: + def config = new NomadConfig( + client:[ + address : "http://${mockWebServer.hostName}:${mockWebServer.port}" + ], + jobs: [ + secrets:[ + enable: true, + path: 'test' + ] + ] + ) + def service = new NomadService(config) + + when: + mockWebServer.enqueue(new MockResponse() + .addHeader("Content-Type", "application/json")); + + service.setVariableValue("MySecret", "MyValue") + def recordedRequest = mockWebServer.takeRequest(); + + then: + recordedRequest.method == "POST" + recordedRequest.path == "/v1/var/test%2FMySecret" + } + +} diff --git a/settings.gradle b/settings.gradle index 832c3d6..42569db 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,5 +18,3 @@ rootProject.name = 'nf-nomad' include 'plugins' include('plugins:nf-nomad') - -//includeBuild('_resources/nextflow') diff --git a/validation/secrets/main.nf b/validation/secrets/main.nf new file mode 100644 index 0000000..82259b8 --- /dev/null +++ b/validation/secrets/main.nf @@ -0,0 +1,23 @@ +#!/usr/bin/env nextflow + +process sayHello { + container 'ubuntu:20.04' + secret 'MY_ACCESS_KEY' + secret 'MY_SECRET_KEY' + + input: + val x + output: + stdout + + """ + echo $x world! the access \$MY_ACCESS_KEY and the secret \$MY_SECRET_KEY + """ +} + +workflow { + Channel.of('Bonjour', 'Ciao', 'Hello', 'Hola') | sayHello | view +} +workflow.onComplete { + println("The secret is: ${secrets.MY_ACCESS_KEY}") +} \ No newline at end of file diff --git a/validation/secrets/nextflow.config b/validation/secrets/nextflow.config new file mode 100644 index 0000000..93e1219 --- /dev/null +++ b/validation/secrets/nextflow.config @@ -0,0 +1,26 @@ +plugins { + id "nf-nomad@${System.getenv("NOMAD_PLUGIN_VERSION") ?: "latest"}" +} + +process { + executor = "nomad" +} + +nomad { + + client { + address = "http://localhost:4646" + } + + jobs { + deleteOnCompletion = false + volume = { type "host" name "scratchdir" } + namespace = 'nf-nomad' + + secrets { + enabled = true //if false then use LocalSecretsProvider implementation + } + } + +} + diff --git a/validation/start-nomad.sh b/validation/start-nomad.sh index 215506f..e63aa8d 100755 --- a/validation/start-nomad.sh +++ b/validation/start-nomad.sh @@ -2,7 +2,14 @@ set -ue NOMAD_VERSION="1.8.1" -NOMAD_PLATFORM="linux_amd64" +NOMAD_PLATFORM=${NOMAD_PLATFORM:-linux_amd64} + +## Available platforms +#- "linux_amd64" +#- "linux_arm64" +#- "darwin_amd64" +#- "darwin_arm64" +#- "windows_amd64" SECURE=0 [[ "$@" =~ '--secure' ]] && SECURE=1 @@ -10,7 +17,8 @@ SECURE=0 if [ ! -f ./nomad ]; then curl -O "https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip" unzip nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip - rm -f nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip + rm -f nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip LICENSE.txt + chmod +x ./nomad fi mkdir -p nomad_temp @@ -53,13 +61,14 @@ if [ "$SECURE" == 0 ]; then # basic nomad cluter ../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf else - # secured nomad cluster +# secured nomad cluster ../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf & cd .. +#./nomad namespace apply -description "local-nomadlab" nf-nomad ./wait-nomad.sh sleep 3 NOMAD_TOKEN=$(nomad acl bootstrap | awk '/^Secret ID/ {print $4}') export NOMAD_TOKEN echo New super token generated. echo export NOMAD_TOKEN=$NOMAD_TOKEN -fi +fi \ No newline at end of file diff --git a/validation/stop-nomad.sh b/validation/stop-nomad.sh index 85f2ae1..714b400 100755 --- a/validation/stop-nomad.sh +++ b/validation/stop-nomad.sh @@ -5,4 +5,5 @@ sleep 1 df -h --output=target | grep nf-task | xargs sudo umount pkill -9 nomad sleep 1 -rm -rf nomad_temp \ No newline at end of file +rm -rf nomad_temp +rm ./nomad diff --git a/validation/sun-nomadlab/nextflow.config b/validation/sun-nomadlab/nextflow.config index eed649f..5e5de70 100644 --- a/validation/sun-nomadlab/nextflow.config +++ b/validation/sun-nomadlab/nextflow.config @@ -7,8 +7,14 @@ process { } aws { +// From Nomad variables secret store +// accessKey = secrets.SUN_NOMADLAB_MINIO_ACCESS_KEY +// secretKey = secrets.SUN_NOMADLAB_MINIO_SECRET_KEY + +// From local Nextflow secret store accessKey = secrets.SUN_NOMADLAB_ACCESS_KEY secretKey = secrets.SUN_NOMADLAB_SECRET_KEY + client { endpoint = 'http://100.119.165.23:9000' } @@ -30,18 +36,24 @@ nomad { } jobs { + namespace = 'nf-nomad' + secrets { + enabled = true + } + deleteOnCompletion = false + volume = { type "host" name "scratch" } + constraints = { +// attr { +// unique = [hostname:'nomad03'] +// //raw 'platform.aws.instance-type', '=', 'm4.xlarge' node { unique = [name: "nomad03"] } - attr { - unique = [hostname:'nomad03'] - //raw 'platform.aws.instance-type', '=', 'm4.xlarge' - } } } }