runs dbt utils test with env

dlt-hub · Jan 27, 2023 · 5372a72 · 5372a72
1 parent 58a2ea1
commit 5372a72
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 10 deletions.
diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml
@@ -12,11 +12,12 @@ env:
   # all credentials must be present to be passed to dbt runner
   DESTINATION__POSTGRES__CREDENTIALS: postgres://[email protected]:5432/dlt_data
   DESTINATION__REDSHIFT__CREDENTIALS: postgres://[email protected]:5439/chat_analytics_rasa_ci
-  CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }}
-  CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci
-  CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com
-  CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }}
-  CREDENTIALS__TOKEN_URI: https://oauth2.googleapis.com/token
+  DESTINATION__CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }}
+
+  DESTINATION__CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci
+  DESTINATION__CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com
+  DESTINATION__CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }}
+  DESTINATION__CREDENTIALS__TOKEN_URI: https://oauth2.googleapis.com/token
 
   RUNTIME__LOG_LEVEL: ERROR
 

diff --git a/tests/dbt_runner/cases/profiles.yml b/tests/dbt_runner/cases/profiles.yml
@@ -3,9 +3,9 @@ jaffle_shop:
   outputs:
     analytics:
       type: postgres
-      host: localhost
-      user: loader
-      password: loader
+      host: "{{ env_var('CREDENTIALS__HOST') }}"
+      user: "{{ env_var('CREDENTIALS__USERNAME') }}"
+      password: "{{ env_var('CREDENTIALS__PASSWORD') }}"
       port: 5432
       dbname: dlt_data
       schema: "{{ var('dbt_schema') }}"
diff --git a/tests/dbt_runner/local/test_dbt_utils.py b/tests/dbt_runner/local/test_dbt_utils.py
@@ -1,12 +1,15 @@
 import os
 import shutil
 import pytest
+from dlt.common.configuration.resolve import resolve_configuration
+from dlt.common.configuration.specs import PostgresCredentials
+from dlt.common.configuration.utils import add_config_to_env
 from dlt.common.runners.synth_pickle import decode_obj
 
 from dlt.common.storages import FileStorage
 from dlt.common.utils import uniq_id
 from dlt.dbt_runner.dbt_utils import DBTProcessingError, initialize_dbt_logging, run_dbt_command, is_incremental_schema_out_of_sync_error
-from tests.utils import test_storage
+from tests.utils import test_storage, preserve_environ
 from tests.dbt_runner.utils import clone_jaffle_repo, load_test_case
 
 
@@ -22,11 +25,15 @@ def test_dbt_commands(test_storage: FileStorage) -> None:
     # profiles in cases require this var to be set
     dbt_vars = {"dbt_schema": schema_name}
 
+    # extract postgres creds from env, parse and emit
+    credentials = resolve_configuration(PostgresCredentials(),  namespaces=("destination", "postgres"))
+    add_config_to_env(credentials)
+
     repo_path = clone_jaffle_repo(test_storage)
     # copy profile
     shutil.copy("./tests/dbt_runner/cases/profiles_invalid_credentials.yml", os.path.join(repo_path, "profiles.yml"))
     # initialize logging
-    global_args = initialize_dbt_logging("INFO", False)
+    global_args = initialize_dbt_logging("ERROR", False)
     # run deps, results are None
     assert run_dbt_command(repo_path, "deps", ".", global_args=global_args) is None