diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml new file mode 100644 index 00000000..2e352cea --- /dev/null +++ b/integration_tests/profiles.yml @@ -0,0 +1,50 @@ + +# HEY! This file is used in the dbt-utils integrations tests with CircleCI. +# You should __NEVER__ check credentials into version control. That's why we use environment variables everywhere. +# Thanks for reading :) + +config: + send_anonymous_usage_stats: False + use_colors: True + +integration_tests: + target: postgres + outputs: + postgres: + type: postgres + host: "{{ env_var('POSTGRES_TEST_HOST') }}" + user: "{{ env_var('POSTGRES_TEST_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_TEST_PASS') }}" + port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}" + dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}" + schema: "{{ env_var('POSTGRES_TEST_SCHEMA') }}" + threads: 5 + + redshift: + type: redshift + host: "{{ env_var('REDSHIFT_TEST_HOST') }}" + user: "{{ env_var('REDSHIFT_TEST_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_TEST_PASS') }}" + dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}" + port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}" + schema: "{{ env_var('REDSHIFT_TEST_SCHEMA') }}" + threads: 5 + + bigquery: + type: bigquery + method: service-account + keyfile: "{{ env_var('DBT_ENV_SECRET_BIGQUERY_SERVICE_KEY_PATH') }}" + project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" + schema: "{{ env_var('BIGQUERY_TEST_SCHEMA') }}" + threads: 10 + + snowflake: + type: snowflake + account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_TEST_USER') }}" + password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_PASS') }}" + role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}" + database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" + warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" + schema: "{{ env_var('SNOWFLAKE_TEST_SCHEMA') }}" + threads: 10 diff --git a/tox.ini b/tox.ini index 8dfabb62..4659e200 100644 --- a/tox.ini +++ b/tox.ini @@ -5,59 +5,84 @@ envlist = lint_all, textenv [testenv] passenv = DBT_PROFILES_DIR - ; postgres env vars + # postgres env vars POSTGRES_TEST_HOST POSTGRES_TEST_USER DBT_ENV_SECRET_POSTGRES_TEST_PASS POSTGRES_TEST_PORT POSTGRES_TEST_DBNAME POSTGRES_TEST_SCHEMA - ; snowflake env vars + # snowflake env vars SNOWFLAKE_TEST_ACCOUNT SNOWFLAKE_TEST_USER - DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD + DBT_ENV_SECRET_SNOWFLAKE_TEST_PASS SNOWFLAKE_TEST_ROLE SNOWFLAKE_TEST_DATABASE SNOWFLAKE_TEST_WAREHOUSE SNOWFLAKE_TEST_SCHEMA +# Uses pytest to run tests, very basic and doesn't require dbt +[testenv:dbt_integration_pytest] +changedir = {toxinidir} +allowlist_externals = pytest +deps = pytest +skip_install = true +commands = + pytest {posargs} tests/functional # Snowflake integration tests for centralized dbt testing -[testenv:centralized_integration_snowflake] +# run dbt commands directly, assumes dbt is already installed in environment +# tox runs at the root of the repo, so we need to cd out of the integration_tests directory +[testenv:dbt_integration_snowflake] changedir = integration_tests -allowlist_externals = dbt +allowlist_externals = + dbt + bash + {[testenv:dbt_integration_pytest]allowlist_externals} skip_install = true +deps = {[testenv:dbt_integration_pytest]deps} commands = dbt --version dbt debug --target snowflake dbt deps --target snowflake dbt build --target snowflake --full-refresh + bash -c "cd {toxinidir} && {[testenv:dbt_integration_pytest]commands}" # Postgres integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment # tox runs at the root of the repo, so we need to cd out of the integration_tests directory -[testenv:centralized_integration_postgres] +[testenv:dbt_integration_postgres] changedir = integration_tests allowlist_externals = dbt - bash - {[testenv:centralized_integration_postgres_pytest]allowlist_externals} skip_install = true -deps = {[testenv:centralized_integration_postgres_pytest]deps} commands = dbt --version dbt debug --target postgres dbt deps --target postgres dbt build --target postgres --full-refresh - bash -c "cd {toxinidir} && {[testenv:centralized_integration_postgres_pytest]commands}" -# Postgres integration tests for centralized dbt testing -# Uses pytest to run tests, assumes dbt is already installed in environment -[testenv:centralized_integration_postgres_pytest] -changedir = {toxinidir} -allowlist_externals = pytest -deps = pytest +# BigQuery integration tests for centralized dbt testing +[testenv:dbt_integration_bigquery] +changedir = integration_tests +allowlist_externals = + dbt skip_install = true commands = - pytest {posargs} tests/functional + dbt --version + dbt debug --target postgres + dbt deps --target postgres + dbt build --target postgres --full-refresh + +# redshift integration tests for centralized dbt testing +[testenv:dbt_integration_redshift] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --version + dbt debug --target postgres + dbt deps --target postgres + dbt build --target postgres --full-refresh \ No newline at end of file