basic cleanup

integration_tests/profiles.yml

@@ -0,0 +1,50 @@
+
+# HEY! This file is used in the dbt-utils integrations tests with CircleCI.
+# You should __NEVER__ check credentials into version control. That's why we use environment variables everywhere.
+# Thanks for reading :)
+
+config:
+    send_anonymous_usage_stats: False
+    use_colors: True
+
+integration_tests:
+  target: postgres
+  outputs:
+    postgres:
+      type: postgres
+      host: "{{ env_var('POSTGRES_TEST_HOST') }}"
+      user: "{{ env_var('POSTGRES_TEST_USER') }}"
+      pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_TEST_PASS') }}"
+      port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}"
+      dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}"
+      schema: "{{ env_var('POSTGRES_TEST_SCHEMA') }}"
+      threads: 5
+
+    redshift:
+      type: redshift
+      host: "{{ env_var('REDSHIFT_TEST_HOST') }}"
+      user: "{{ env_var('REDSHIFT_TEST_USER') }}"
+      pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_TEST_PASS') }}"
+      dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}"
+      port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}"
+      schema: "{{ env_var('REDSHIFT_TEST_SCHEMA') }}"
+      threads: 5
+
+    bigquery:
+      type: bigquery
+      method: service-account
+      keyfile: "{{ env_var('DBT_ENV_SECRET_BIGQUERY_SERVICE_KEY_PATH') }}"
+      project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}"
+      schema: "{{ env_var('BIGQUERY_TEST_SCHEMA') }}"
+      threads: 10
+
+    snowflake:
+      type: snowflake
+      account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}"
+      user: "{{ env_var('SNOWFLAKE_TEST_USER') }}"
+      password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_PASS') }}"
+      role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}"
+      database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}"
+      warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}"
+      schema: "{{ env_var('SNOWFLAKE_TEST_SCHEMA') }}"
+      threads: 10

tox.ini

@@ -5,59 +5,84 @@ envlist = lint_all, textenv
 [testenv]
 passenv =
     DBT_PROFILES_DIR
-    ; postgres env vars
+    # postgres env vars
     POSTGRES_TEST_HOST
     POSTGRES_TEST_USER
     DBT_ENV_SECRET_POSTGRES_TEST_PASS
     POSTGRES_TEST_PORT
     POSTGRES_TEST_DBNAME
     POSTGRES_TEST_SCHEMA
-    ; snowflake env vars
+    # snowflake env vars
     SNOWFLAKE_TEST_ACCOUNT
     SNOWFLAKE_TEST_USER
-    DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD
+    DBT_ENV_SECRET_SNOWFLAKE_TEST_PASS
     SNOWFLAKE_TEST_ROLE
     SNOWFLAKE_TEST_DATABASE
     SNOWFLAKE_TEST_WAREHOUSE
     SNOWFLAKE_TEST_SCHEMA
 
+# Uses pytest to run tests, very basic and doesn't require dbt
+[testenv:dbt_integration_pytest]
+changedir = {toxinidir}
+allowlist_externals = pytest
+deps = pytest
+skip_install = true
+commands =
+      pytest {posargs} tests/functional
 
 # Snowflake integration tests for centralized dbt testing
-[testenv:centralized_integration_snowflake]
+# run dbt commands directly, assumes dbt is already installed in environment
+# tox runs at the root of the repo, so we need to cd out of the integration_tests directory
+[testenv:dbt_integration_snowflake]
 changedir = integration_tests
-allowlist_externals = dbt
+allowlist_externals = 
+    dbt
+    bash
+    {[testenv:dbt_integration_pytest]allowlist_externals}
 skip_install = true
+deps = {[testenv:dbt_integration_pytest]deps}
 commands =
     dbt --version
     dbt debug --target snowflake
     dbt deps --target snowflake
     dbt build --target snowflake --full-refresh
+    bash -c "cd {toxinidir} && {[testenv:dbt_integration_pytest]commands}"
 
 
 # Postgres integration tests for centralized dbt testing
 # run dbt commands directly, assumes dbt is already installed in environment
 # tox runs at the root of the repo, so we need to cd out of the integration_tests directory
-[testenv:centralized_integration_postgres]
+[testenv:dbt_integration_postgres]
 changedir = integration_tests
 allowlist_externals = 
     dbt
-    bash
-    {[testenv:centralized_integration_postgres_pytest]allowlist_externals}
 skip_install = true
-deps = {[testenv:centralized_integration_postgres_pytest]deps}
 commands =
     dbt --version
     dbt debug --target postgres
     dbt deps --target postgres
     dbt build --target postgres --full-refresh
-    bash -c "cd {toxinidir} && {[testenv:centralized_integration_postgres_pytest]commands}"
 
-# Postgres integration tests for centralized dbt testing
-# Uses pytest to run tests, assumes dbt is already installed in environment
-[testenv:centralized_integration_postgres_pytest]
-changedir = {toxinidir}
-allowlist_externals = pytest
-deps = pytest
+# BigQuery integration tests for centralized dbt testing
+[testenv:dbt_integration_bigquery]
+changedir = integration_tests
+allowlist_externals = 
+    dbt
 skip_install = true
 commands =
-      pytest {posargs} tests/functional
+    dbt --version
+    dbt debug --target postgres
+    dbt deps --target postgres
+    dbt build --target postgres --full-refresh
+
+# redshift integration tests for centralized dbt testing
+[testenv:dbt_integration_redshift]
+changedir = integration_tests
+allowlist_externals = 
+    dbt
+skip_install = true
+commands =
+    dbt --version
+    dbt debug --target postgres
+    dbt deps --target postgres
+    dbt build --target postgres --full-refresh