diff --git a/.circleci/config.yml b/.circleci/config.yml index 2585c60..6746196 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: - checkout - run: - run: setup_creds + name: setup_creds command: | echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json @@ -29,11 +29,12 @@ jobs: - run: name: "Run Tests - Postgres" environment: - POSTGRES_TEST_HOST: localhost - POSTGRES_TEST_USER: root - POSTGRES_TEST_PASS: "" - POSTGRES_TEST_PORT: 5432 - POSTGRES_TEST_DBNAME: circle_test + POSTGRES_HOST: localhost + POSTGRES_USER: root + DBT_ENV_SECRET_POSTGRES_PASS: "" + POSTGRES_PORT: 5432 + POSTGRES_DATABASE: circle_test + POSTGRES_SCHEMA: codegen_integration_tests_postgres command: | . dbt_venv/bin/activate cd integration_tests diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6a8d288 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,33 @@ +# **what?** +# Run tests for dbt-codegen against supported adapters + +# **why?** +# To ensure that dbt-codegen works as expected with all supported adapters + +# **when?** +# On every PR, and every push to main and when manually triggered + +name: Package Integration Tests + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + run-tests: + uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1 + # this just tests with postgres so no variables need to be passed through. + # When it's time to add more adapters you will need to pass through inputs for + # the other adapters as shown in the below example for redshift + # with: + # # redshift + # REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} + # REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} + # REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} + # REDSHIFT_SCHEMA: "integration_tests_redshift_${{ github.run_number }}" + # REDSHIFT_PORT: ${{ vars.REDSHIFT_PORT }} + # secrets: + # DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.DBT_ENV_SECRET_REDSHIFT_PASS }} diff --git a/Makefile b/Makefile index 9472484..e3545fc 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,11 @@ test: ## Run the integration tests. @./run_test.sh $(target) +.PHONY: test_tox +test: ## Run the integration tests with tox + @\ + tox -e dbt_integration_$(target) + .PHONY: dev dev: ## Installs dbt-* packages in develop mode along with development dependencies. @\ diff --git a/integration_tests/.env/bigquery.env b/integration_tests/.env/bigquery.env new file mode 100644 index 0000000..4972fca --- /dev/null +++ b/integration_tests/.env/bigquery.env @@ -0,0 +1,3 @@ +BIGQUERY_PROJECT= +BIGQUERY_SCHEMA= +BIGQUERY_KEYFILE_JSON= diff --git a/integration_tests/.env/postgres.env b/integration_tests/.env/postgres.env new file mode 100644 index 0000000..4fd155c --- /dev/null +++ b/integration_tests/.env/postgres.env @@ -0,0 +1,6 @@ +POSTGRES_HOST=localhost +POSTGRES_USER=root +DBT_ENV_SECRET_POSTGRES_PASS=password +POSTGRES_PORT=5432 +POSTGRES_DATABASE=codegen_test +POSTGRES_SCHEMA=codegen_integration_tests_postgres diff --git a/integration_tests/.env/redshift.env b/integration_tests/.env/redshift.env new file mode 100644 index 0000000..baf768c --- /dev/null +++ b/integration_tests/.env/redshift.env @@ -0,0 +1,5 @@ +REDSHIFT_TEST_HOST= +REDSHIFT_TEST_USER= +REDSHIFT_TEST_PASS= +REDSHIFT_TEST_DBNAME= +REDSHIFT_TEST_PORT= diff --git a/integration_tests/.env/snowflake.env b/integration_tests/.env/snowflake.env new file mode 100644 index 0000000..eb10da9 --- /dev/null +++ b/integration_tests/.env/snowflake.env @@ -0,0 +1,6 @@ +SNOWFLAKE_TEST_ACCOUNT= +SNOWFLAKE_TEST_USER= +SNOWFLAKE_TEST_PASSWORD= +SNOWFLAKE_TEST_ROLE= +SNOWFLAKE_TEST_DATABASE= +SNOWFLAKE_TEST_WAREHOUSE= diff --git a/integration_tests/README.md b/integration_tests/README.md index 1337e57..ed9c863 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -44,35 +44,7 @@ You can set these env vars in a couple ways: - **Temporary**: Set these environment variables in your shell before running the tests. This is the easiest way to get started, but you'll have to set them every time you open a new terminal. - **Reusable**: If you anticipate developing for multiple sessions, set these environment variables in your shell profile (like `~/.bashrc` or `~/.zshrc`). This way, you won't have to set them every time you open a new terminal. -The environment variables you'll need to set for each adapter are: - -```bash -# Postgres — these are the defaults for the Docker container so actually have values -export POSTGRES_TEST_HOST=localhost -export POSTGRES_TEST_USER=root -export POSTGRES_TEST_PASS='' -export POSTGRES_TEST_PORT=5432 -export POSTGRES_TEST_DBNAME=circle_test - -# BigQuery -export BIGQUERY_SERVICE_KEY_PATH= -export BIGQUERY_TEST_DATABASE= - -# Redshift -export REDSHIFT_TEST_HOST= -export REDSHIFT_TEST_USER= -export REDSHIFT_TEST_PASS= -export REDSHIFT_TEST_DBNAME= -export REDSHIFT_TEST_PORT= - -# Snowflake -export SNOWFLAKE_TEST_ACCOUNT= -export SNOWFLAKE_TEST_USER= -export SNOWFLAKE_TEST_PASSWORD= -export SNOWFLAKE_TEST_ROLE= -export SNOWFLAKE_TEST_DATABASE= -export SNOWFLAKE_TEST_WAREHOUSE= -``` +The environment variables you'll need to set for each adapter can be found in [integration_tests/.env/](integration_tests/.env/). ### Setup Postgres or other database targets @@ -139,7 +111,9 @@ source .venv/bin/activate ## Write or modify an integration test -### Run the integration tests +Run all the tests _before_ you start developing to make sure everything is working as expected before you start making changes. Nothing is worse than spending a ton of time troubleshooting a failing test, only to realize it was failing before you touched anything. This will also ensure that you have the correct environment variables set up and that your database is running. + +### Run the Circle CI integration tests To run all the integration tests on your local machine like they will get run in CI: @@ -157,7 +131,13 @@ make test target=postgres ./run_test.sh postgres ``` -Run all the tests _before_ you start developing to make sure everything is working as expected before you start making changes. Nothing is worse than spending a ton of time troubleshooting a failing test, only to realize it was failing before you touched anything. This will also ensure that you have the correct environment variables set up and that your database is running. +### Run the tox Supported Tests + +To run all the integration tests on your local machine like they will get run in the CI (using GitHub workflows with tox): + +```shell +make test_tox target=postgres +``` ### Creating a new integration test diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 6f194d0..7d0c8f2 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -15,6 +15,10 @@ clean-targets: - "target" - "dbt_packages" +flags: + send_anonymous_usage_stats: False + use_colors: True + seeds: +schema: raw_data +quote_columns: false diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index ffe7a71..0db3973 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -1,23 +1,20 @@ -# HEY! This file is used in the dbt-codegen integrations tests with CircleCI. -# You should __NEVER__ check credentials into version control. Thanks for reading :) - -config: - send_anonymous_usage_stats: False - use_colors: True +# HEY! This file is used in the dbt-codegen integrations tests with GitHub CI. +# You should __NEVER__ check credentials into version control. That's why we use environment variables everywhere. +# Thanks for reading :) integration_tests: target: postgres outputs: postgres: - type: postgres - host: "{{ env_var('POSTGRES_TEST_HOST') }}" - user: "{{ env_var('POSTGRES_TEST_USER') }}" - pass: "{{ env_var('POSTGRES_TEST_PASS') }}" - port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}" - dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}" - schema: codegen_integration_tests_postgres - threads: 1 + type: "postgres" + host: "{{ env_var('POSTGRES_HOST') }}" + user: "{{ env_var('POSTGRES_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" + port: "{{ env_var('POSTGRES_PORT') | as_number }}" + dbname: "{{ env_var('POSTGRES_DATABASE') }}" + schema: "{{ env_var('POSTGRES_SCHEMA') }}" + threads: 5 redshift: type: redshift diff --git a/supported_adapters.env b/supported_adapters.env new file mode 100644 index 0000000..79c9a19 --- /dev/null +++ b/supported_adapters.env @@ -0,0 +1 @@ +SUPPORTED_ADAPTERS=postgres diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..9918e82 --- /dev/null +++ b/tox.ini @@ -0,0 +1,27 @@ +[tox] +skipsdist = True +envlist = lint_all, testenv + +[testenv] +passenv = + # postgres env vars + POSTGRES_HOST + POSTGRES_USER + DBT_ENV_SECRET_POSTGRES_PASS + POSTGRES_PORT + POSTGRES_DATABASE + POSTGRES_SCHEMA + +# Postgres integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_postgres] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --warn-error deps --target postgres + dbt --warn-error run-operation create_source_table --target postgres + dbt --warn-error seed --target postgres --full-refresh + dbt --warn-error run --target postgres + dbt --warn-error test --target postgres