diff --git a/data/.env.template b/data/.env.template index bac0cdb2..dd663786 100644 --- a/data/.env.template +++ b/data/.env.template @@ -1,3 +1,6 @@ +# Developer Prefix +USER_PREFIX=PNADOLNY + # Google Analytics Tap TAP_GOOGLE_ANALYTICS_CLIENT_SECRETS="{\"type\": \"service_account\",\"project_id\": \"foo\",\"private_key_id\": \"bar\",\"private_key\": \"-----BEGIN PRIVATE KEY-----\\nfoo\\nbar\\n-----END PRIVATE KEY-----\\n\",\"client_email\": \"foo@bar.iam.gserviceaccount.com\",\"client_id\": \"foo\",\"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\"token_uri\": \"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/foobar.iam.gserviceaccount.com\"}" @@ -17,8 +20,5 @@ PERMISSION_BOT_ACCOUNT="****" # Snowflake MELTANO User Password SNOWFLAKE_PASSWORD="****" -# dbt Snowflake Password -DBT_SNOWFLAKE_PASSWORD="****" - # Slack Webhook TARGET_APPRISE_URIS=["https://hooks.slack.com/services/{}/{}/{}"] diff --git a/data/environments/userdev.meltano.yml b/data/environments/userdev.meltano.yml index 064ca83f..4932617a 100644 --- a/data/environments/userdev.meltano.yml +++ b/data/environments/userdev.meltano.yml @@ -75,7 +75,6 @@ environments: role: ${USER_PREFIX} warehouse: CORE env: - USER_PREFIX: PNADOLNY HUB_METRICS_S3_PATH: s3://devtest-meltano-bucket-01/hub_metrics/ SUPERSET_API_URL: http://localhost:8088 SUPERSET_USER: admin diff --git a/data/meltano.yml b/data/meltano.yml index 6122bd80..3cf9d05a 100644 --- a/data/meltano.yml +++ b/data/meltano.yml @@ -2,6 +2,7 @@ version: 1 default_environment: userdev send_anonymous_usage_stats: false project_id: c15e971a-d318-4a9d-979b-1039ce5fd1b1 + include_paths: - ./environments/*.meltano.yml - ./extract/*.meltano.yml @@ -9,3 +10,7 @@ include_paths: - ./orchestrate/*.meltano.yml - ./transform/*.meltano.yml - ./utilities/*.meltano.yml + +# Feature Flags - https://docs.meltano.com/reference/settings#feature-flags +# fail if any referenced env vars are missing +ff.strict_env_var_mode: true diff --git a/data/transform/profiles/snowflake/profiles.yml b/data/transform/profiles/snowflake/profiles.yml index c49492f4..12a98713 100644 --- a/data/transform/profiles/snowflake/profiles.yml +++ b/data/transform/profiles/snowflake/profiles.yml @@ -13,7 +13,7 @@ meltano: threads: 2 account: "{{ env_var('DBT_SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('DBT_SNOWFLAKE_USER') }}" - password: "{{ env_var('SNOWFLAKE_PASSWORD') }}" + password: "{{ env_var('DBT_SNOWFLAKE_PASSWORD') }}" role: "{{ env_var('DBT_SNOWFLAKE_ROLE') }}" warehouse: "{{ env_var('DBT_SNOWFLAKE_WAREHOUSE') }}" database: "{{ env_var('DBT_SNOWFLAKE_DATABASE') }}" @@ -23,7 +23,7 @@ meltano: threads: 2 account: "{{ env_var('DBT_SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('DBT_SNOWFLAKE_USER') }}" - password: "{{ env_var('SNOWFLAKE_PASSWORD') }}" + password: "{{ env_var('DBT_SNOWFLAKE_PASSWORD') }}" role: "{{ env_var('DBT_SNOWFLAKE_ROLE') }}" warehouse: "{{ env_var('DBT_SNOWFLAKE_WAREHOUSE') }}" database: "{{ env_var('DBT_SNOWFLAKE_DATABASE') }}" @@ -33,7 +33,7 @@ meltano: threads: 4 account: "{{ env_var('DBT_SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('DBT_SNOWFLAKE_USER') }}" - password: "{{ env_var('SNOWFLAKE_PASSWORD') }}" + password: "{{ env_var('DBT_SNOWFLAKE_PASSWORD') }}" role: "{{ env_var('DBT_SNOWFLAKE_ROLE') }}" warehouse: "{{ env_var('DBT_SNOWFLAKE_WAREHOUSE') }}" database: "{{ env_var('DBT_SNOWFLAKE_DATABASE') }}" @@ -43,7 +43,7 @@ meltano: threads: 6 account: "{{ env_var('DBT_SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('DBT_SNOWFLAKE_USER') }}" - password: "{{ env_var('SNOWFLAKE_PASSWORD') }}" + password: "{{ env_var('DBT_SNOWFLAKE_PASSWORD') }}" role: "{{ env_var('DBT_SNOWFLAKE_ROLE') }}" warehouse: "{{ env_var('DBT_SNOWFLAKE_WAREHOUSE') }}" database: "{{ env_var('DBT_SNOWFLAKE_DATABASE') }}" @@ -53,7 +53,7 @@ meltano: threads: 4 account: "{{ env_var('DBT_SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('DBT_SNOWFLAKE_USER') }}" - password: "{{ env_var('SNOWFLAKE_PASSWORD') }}" + password: "{{ env_var('DBT_SNOWFLAKE_PASSWORD') }}" role: "{{ env_var('DBT_SNOWFLAKE_ROLE') }}" warehouse: "{{ env_var('DBT_SNOWFLAKE_WAREHOUSE') }}" database: "{{ env_var('DBT_SNOWFLAKE_DATABASE') }}" diff --git a/data/transform/transformers.meltano.yml b/data/transform/transformers.meltano.yml index 79cb4462..76c51671 100644 --- a/data/transform/transformers.meltano.yml +++ b/data/transform/transformers.meltano.yml @@ -49,3 +49,4 @@ plugins: account: epa06486 database_raw: RAW database_prep: PREP + password: ${SNOWFLAKE_PASSWORD} diff --git a/data/utilities/utilities.meltano.yml b/data/utilities/utilities.meltano.yml index 401f632b..3c828a7c 100644 --- a/data/utilities/utilities.meltano.yml +++ b/data/utilities/utilities.meltano.yml @@ -7,6 +7,8 @@ plugins: settings: - name: user env: DBT_SNOWFLAKE_USER + - name: password + env: DBT_SNOWFLAKE_PASSWORD - name: role env: DBT_SNOWFLAKE_ROLE - name: database @@ -26,6 +28,7 @@ plugins: account: epa06486 role: TRANSFORMER warehouse: CORE + password: ${SNOWFLAKE_PASSWORD} - name: awscli namespace: awscli pip_url: awscli==1.21.7