chore: Use secrets manager function for secrets.

edx · Mar 25, 2024 · 787570f · 787570f
1 parent 3a4d90a
commit 787570f
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 5 deletions.
diff --git a/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy b/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy
@@ -82,6 +82,7 @@ class SnowflakeRefreshSnowpipe {
                 }
                 publishers common_publishers(allVars)
                 steps {
+                    shell(dslFactory.readFileFromWorkspace('dataeng/resources/secrets-manager-setup.sh'))
                     shell(dslFactory.readFileFromWorkspace('dataeng/resources/snowflake-refresh-snowpipe.sh'))
                 }
             }

diff --git a/dataeng/resources/snowflake-refresh-snowpipe.sh b/dataeng/resources/snowflake-refresh-snowpipe.sh
@@ -10,11 +10,15 @@ source "${PYTHON_VENV}/bin/activate"
 cd $WORKSPACE/analytics-tools/snowflake
 make requirements
 
-source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS KEY_PATH
-source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS PASSPHRASE_PATH
-source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS USER
-source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS ACCOUNT
-source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS JOB_FREQUENCY
+# Source the secrets-manager.sh script to make the function available
+source $WORKSPACE/secrets-manager.sh
+# Fetch the secrets from AWS
+set +x
+get_secret_value analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS KEY_PATH
+get_secret_value analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS PASSPHRASE_PATH
+get_secret_value analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS USER
+get_secret_value analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS ACCOUNT
+set -x
 
 python refresh_snowpipe.py \
     --key_path $WORKSPACE/analytics-secure/$KEY_PATH \
-Original file line number
+Diff line change
@@ Expand Up / @@ -82,6 +82,7 @@ class SnowflakeRefreshSnowpipe { @@
                     }
                     publishers common_publishers(allVars)
                     steps {
+                        shell(dslFactory.readFileFromWorkspace('dataeng/resources/secrets-manager-setup.sh'))
                         shell(dslFactory.readFileFromWorkspace('dataeng/resources/snowflake-refresh-snowpipe.sh'))
                     }
                 }
@@ Expand Down @@