From 0180d47b05fe6fadd54d5b7b3c748235eff504c9 Mon Sep 17 00:00:00 2001 From: Jorge Aguilera Date: Wed, 28 Aug 2024 11:01:52 +0200 Subject: [PATCH] improve secrets provider Signed-off-by: Jorge Aguilera --- .../nomad/executor/NomadService.groovy | 3 +- .../nomad/secrets/NomadSecretProvider.groovy | 10 +++++ validation/install-nomad.sh | 20 ++++++++++ validation/run-all.sh | 2 + validation/secrets/nextflow.config | 5 +++ validation/start-nomad.sh | 40 +++++++------------ validation/stop-nomad.sh | 3 +- 7 files changed, 54 insertions(+), 29 deletions(-) create mode 100755 validation/install-nomad.sh diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy index 080e172..40968e5 100644 --- a/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy +++ b/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy @@ -417,7 +417,8 @@ class NomadService implements Closeable{ config.jobOpts().region, config.jobOpts().namespace, null, null, null, null, null, null, null) - listRequest.collect{ it.path} + String path = (config.jobOpts().secretOpts?.path ?: '')+"/" + listRequest.collect{ it.path - path} } void deleteVariable(String key){ diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretProvider.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretProvider.groovy index 0375ec2..93078e5 100644 --- a/plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretProvider.groovy +++ b/plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretProvider.groovy @@ -26,6 +26,16 @@ class NomadSecretProvider extends LocalSecretsProvider implements SecretsProvide return super.load() } + @Override + protected List loadSecrets() { + Set names = listSecretsNames() + List ret = names.collect{ name-> + String value = getSecret(name) + new SecretImpl(name, value) + } + ret + } + protected boolean isEnabled(){ if( !config ){ config = new NomadConfig(Global.config?.nomad as Map ?: Map.of()) diff --git a/validation/install-nomad.sh b/validation/install-nomad.sh new file mode 100755 index 0000000..b1723db --- /dev/null +++ b/validation/install-nomad.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -ue + +NOMAD_VERSION="1.8.1" +NOMAD_PLATFORM=${NOMAD_PLATFORM:-linux_amd64} + +## Available platforms +#- "linux_amd64" +#- "linux_arm64" +#- "darwin_amd64" +#- "darwin_arm64" +#- "windows_amd64" + + +if [ ! -f ./nomad ]; then + curl -O "https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip" + unzip nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip + rm -f nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip LICENSE.txt + chmod +x ./nomad +fi diff --git a/validation/run-all.sh b/validation/run-all.sh index f1136c0..c5d5297 100755 --- a/validation/run-all.sh +++ b/validation/run-all.sh @@ -46,6 +46,8 @@ if [ "$SKIPLOCAL" == 0 ]; then -profile test,docker --outdir $(pwd)/nomad_temp/scratchdir/bactopia/outdir \ --datasets_cache $(pwd)/nomad_temp/scratchdir/bactopia/datasets + ./run-pipeline.sh -c secrets/nextflow.config secrets/main.nf + else echo "skip local" fi diff --git a/validation/secrets/nextflow.config b/validation/secrets/nextflow.config index 93e1219..e7cac05 100644 --- a/validation/secrets/nextflow.config +++ b/validation/secrets/nextflow.config @@ -6,6 +6,11 @@ process { executor = "nomad" } +aws { + accessKey = secrets.MY_ACCESS_KEY + secretKey = secrets.MY_SECRET_KEY +} + nomad { client { diff --git a/validation/start-nomad.sh b/validation/start-nomad.sh index e63aa8d..cbad1db 100755 --- a/validation/start-nomad.sh +++ b/validation/start-nomad.sh @@ -1,26 +1,11 @@ #!/bin/bash set -ue -NOMAD_VERSION="1.8.1" -NOMAD_PLATFORM=${NOMAD_PLATFORM:-linux_amd64} - -## Available platforms -#- "linux_amd64" -#- "linux_arm64" -#- "darwin_amd64" -#- "darwin_arm64" -#- "windows_amd64" +./install-nomad.sh SECURE=0 [[ "$@" =~ '--secure' ]] && SECURE=1 -if [ ! -f ./nomad ]; then - curl -O "https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip" - unzip nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip - rm -f nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip LICENSE.txt - chmod +x ./nomad -fi - mkdir -p nomad_temp cd nomad_temp @@ -59,16 +44,19 @@ cp ../client.conf . if [ "$SECURE" == 0 ]; then # basic nomad cluter - ../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf + ../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf & else -# secured nomad cluster -../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf & + # secured nomad cluster + ../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf & + sleep 3 + NOMAD_TOKEN=$(../nomad acl bootstrap | awk '/^Secret ID/ {print $4}') + export NOMAD_TOKEN + echo New super token generated. + echo export NOMAD_TOKEN=$NOMAD_TOKEN +fi + cd .. -#./nomad namespace apply -description "local-nomadlab" nf-nomad ./wait-nomad.sh -sleep 3 -NOMAD_TOKEN=$(nomad acl bootstrap | awk '/^Secret ID/ {print $4}') -export NOMAD_TOKEN -echo New super token generated. -echo export NOMAD_TOKEN=$NOMAD_TOKEN -fi \ No newline at end of file +./nomad namespace apply -description "local-nomadlab" nf-nomad +./nomad var put -namespace=nf-nomad secrets/nf-nomad/MY_ACCESS_KEY MY_ACCESS_KEY=TheAccessKey +./nomad var put -namespace=nf-nomad secrets/nf-nomad/MY_SECRET_KEY MY_SECRET_KEY=TheSecretKey \ No newline at end of file diff --git a/validation/stop-nomad.sh b/validation/stop-nomad.sh index 714b400..fade2a4 100755 --- a/validation/stop-nomad.sh +++ b/validation/stop-nomad.sh @@ -3,7 +3,6 @@ ./nomad system gc sleep 1 df -h --output=target | grep nf-task | xargs sudo umount -pkill -9 nomad +kill $(ps aux | grep '../nomad agent' | awk '{print $2}') sleep 1 rm -rf nomad_temp -rm ./nomad