Merge branch 'main' into patch-1

dbt-labs · Sep 28, 2024 · ea7c255 · ea7c255
2 parents 8f2422c + f13a392
commit ea7c255
Show file tree

Hide file tree

Showing 30 changed files with 325 additions and 198 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -10,11 +10,12 @@ jobs:
         environment:
           POSTGRES_USER: root
     environment:
-      POSTGRES_TEST_HOST: localhost
-      POSTGRES_TEST_USER: root
-      POSTGRES_TEST_PASS: ''
-      POSTGRES_TEST_PORT: 5432
-      POSTGRES_TEST_DBNAME: circle_test
+      POSTGRES_HOST: localhost
+      POSTGRES_USER: root
+      DBT_ENV_SECRET_POSTGRES_PASS: ''
+      POSTGRES_PORT: 5432
+      POSTGRES_DATABASE: circle_test
+      POSTGRES_SCHEMA: dbt_utils_integration_tests_postgres
 
     steps:
       - checkout
@@ -71,8 +72,8 @@ jobs:
       - checkout
       - run: pip install --pre dbt-bigquery -r dev-requirements.txt
       - run:
-          name: "Set up credentials"
-          command: echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json
+          name: Setup Environment Variables
+          command: echo 'export BIGQUERY_KEYFILE_JSON="$BIGQUERY_SERVICE_ACCOUNT_JSON"' >> "$BASH_ENV"
       - run:
           name: "Run OG Tests - BigQuery"
           command: ./run_test.sh bigquery
@@ -87,7 +88,8 @@ workflows:
   version: 2
   test-all:
     jobs:
-      - integration-postgres
+      - integration-postgres:
+          context: profile-postgres
       - integration-redshift:
           context: profile-redshift
           requires:

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,44 @@
+# **what?**
+# Run tests for dbt-utils against supported adapters
+
+# **why?**
+# To ensure that dbt-utils works as expected with all supported adapters
+
+# **when?**
+# On every PR, and every push to main and when manually triggered
+
+name: Package Integration Tests
+
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+    workflow_dispatch:
+
+
+jobs:
+  run-tests:
+      uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1
+      with:
+        # no need to pass postgres vars in.  We can just use the defaults in the local container
+        # redshift
+        REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
+        REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
+        REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }}
+        REDSHIFT_SCHEMA: "dbt_utils_integration_tests_redshift_${{ github.run_number }}"
+        REDSHIFT_PORT: 5439
+        # bigquery
+        BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
+        BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery_${{ github.run_number }}"
+        # snowflake
+        SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
+        SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
+        SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
+        SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
+        SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}"
+      secrets:
+        DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }}
+        BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}
+        SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+        DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }}
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,17 @@ dbt_modules/
 dbt_packages/
 logs/
 venv/
-env/
+__pycache__
+.tox/
+/.pytest_cache/
+
+
+# Ignore all directories that start with 'env-' and can have any name after
+env*/
+
+# Do not ignore .env files in any directory and do not ignore .env directories
+!.env
+!*/.env/
+
+# But explicitly ignore test.env files
 test.env
-__pycache__
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,7 +6,50 @@
 - databricks__get_table_types_sql() will now handle streaming_tables ([#861](https://github.com/dbt-labs/dbt-utils/issues/861))
 - databricks__get_table_types_sql() will now correctly handle materialized views ([#881](https://github.com/dbt-labs/dbt-utils/issues/881))
 
-**Full Changelog**: https://github.com/dbt-labs/dbt-utils/compare/1.2.0...main
+**Full Changelog**: https://github.com/dbt-labs/dbt-utils/compare/1.3.0...main
+
+# dbt utils v1.3.0
+
+## What's Changed
+
+### New features
+
+* Add `quote_identifiers` parameter to `unpivot` to handle case-sensitive column names by @error418 in https://github.com/dbt-labs/dbt-utils/pull/792
+
+### Fixes
+
+* Slugify handle empty strings by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/912
+* Update `not_null_proportion` data test to use cross-database `type_numeric()` macro by @henriblancke in https://github.com/dbt-labs/dbt-utils/pull/800
+* Fix `at_least_one` test when `group_by_columns` is configured by @katieclaiborne-duet in https://github.com/dbt-labs/dbt-utils/pull/922
+
+### Documentation
+
+* Remove "I have added an entry to CHANGELOG.md" from the PR template by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/903
+* Add a PR checklist item for "I have read the contributing guide..." as a catch-all by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/905
+* Simplify the PR checklist in relation to testing by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/907
+* Remove the PR checklist items related to the type of change by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/909
+* Align the PR description with dbt-core, dbt-adapters, etc. by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/911
+* Contributors shouldn't edit the `CHANGELOG.md` directly anymore by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/916
+* Contributing guide instructions for allowing commits from maintainers by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/917
+* Link to tests that support `group_by_columns` by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/931
+* Expand description for the `at_least_one` data test by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/933
+* Update release instructions by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/942
+
+### Under the hood
+
+* Move profiles config to project flags by @dbeatty10 in https://github.com/dbt-labs/dbt-utils/pull/926
+* Delete the old unused logo file by @anks2024 in https://github.com/dbt-labs/dbt-utils/pull/936
+* Add tox by @emmyoop in https://github.com/dbt-labs/dbt-utils/pull/919
+
+## New Contributors
+
+* @henriblancke made their first contribution in https://github.com/dbt-labs/dbt-utils/pull/800
+* @katieclaiborne-duet made their first contribution in https://github.com/dbt-labs/dbt-utils/pull/922
+* @anks2024 made their first contribution in https://github.com/dbt-labs/dbt-utils/pull/936
+* @error418 made their first contribution in https://github.com/dbt-labs/dbt-utils/pull/792
+* @emmyoop made their first contribution in https://github.com/dbt-labs/dbt-utils/pull/919
+
+**Full Changelog**: https://github.com/dbt-labs/dbt-utils/compare/1.2.0...1.3.0
 
 # dbt utils v1.2.0
 

diff --git a/Makefile b/Makefile
@@ -2,7 +2,8 @@
 
 .PHONY: test
 test: ## Run the integration tests.
-	@./run_test.sh $(target)
+	@\
+	tox -e dbt_integration_$(target)
 
 .PHONY: dev
 dev: ## Installs dbt-* packages in develop mode along with development dependencies.

diff --git a/README.md b/README.md
@@ -1228,6 +1228,7 @@ Boolean values are replaced with the strings 'true'|'false'
 - `remove`: A list of columns to remove from the resulting table.
 - `field_name`: column name in the resulting table for field
 - `value_name`: column name in the resulting table for value
+- `quote_identifiers` (optional, default=`False`): will encase selected columns and aliases in quotes according to your adapter's implementation of `adapter.quote` (e.g. `"field_name" as "field_name"`).
 
 ### width_bucket ([source](macros/sql/width_bucket.sql))
 

diff --git a/RELEASE.md b/RELEASE.md
@@ -15,7 +15,7 @@ There's a few scenarios that might prompt a release:
 1. Click "Choose a tag", then paste your version number (with no "v" in the name), then click "Create new tag: x.y.z. on publish"
     - The “Release title” will be identical to the tag name
 1. Click the "Generate release notes" button
-1. Copy and paste the generated release notes into `CHANGELOG.md`, commit, and merge into the `main` branch
+1. Copy and paste the generated release notes into [`CHANGELOG.md`](https://github.com/dbt-labs/dbt-utils/blob/main/CHANGELOG.md?plain=1), reformat to match previous entries, commit, and merge into the `main` branch ([example](https://github.com/dbt-labs/dbt-utils/pull/901))
 1. Click the "Publish release" button
     - This will automatically create an "Assets" section containing:
         - Source code (zip)

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -7,3 +7,4 @@ dbt-redshift@git+https://github.com/dbt-labs/dbt-redshift.git
 dbt-snowflake@git+https://github.com/dbt-labs/dbt-snowflake.git
 dbt-bigquery@git+https://github.com/dbt-labs/dbt-bigquery.git
 pytest-xdist
+tox>=3.13
diff --git a/etc/dbt-logo.png b/etc/dbt-logo.png
diff --git a/integration_tests/.env/bigquery.env b/integration_tests/.env/bigquery.env
@@ -1,2 +1,3 @@
-BIGQUERY_SERVICE_KEY_PATH=
-BIGQUERY_TEST_DATABASE=
+BIGQUERY_KEYFILE_JSON=
+BIGQUERY_PROJECT=
+BIGQUERY_SCHEMA=dbt_utils_integration_tests_bigquery
diff --git a/integration_tests/.env/postgres.env b/integration_tests/.env/postgres.env
@@ -1,5 +1,6 @@
-POSTGRES_TEST_HOST=localhost
-POSTGRES_TEST_USER=root
-POSTGRES_TEST_PASS=''
-POSTGRES_TEST_PORT=5432
-POSTGRES_TEST_DBNAME=circle_test
+POSTGRES_HOST=localhost
+POSTGRES_USER=root
+DBT_ENV_SECRET_POSTGRES_PASS=password
+POSTGRES_PORT=5432
+POSTGRES_DATABASE=dbt_utils_test
+POSTGRES_SCHEMA=dbt_utils_integration_tests_postgres
diff --git a/integration_tests/.env/redshift.env b/integration_tests/.env/redshift.env
@@ -1,5 +1,6 @@
-REDSHIFT_TEST_HOST=
-REDSHIFT_TEST_USER=
-REDSHIFT_TEST_PASS=
-REDSHIFT_TEST_DBNAME=
-REDSHIFT_TEST_PORT=
+REDSHIFT_HOST=
+REDSHIFT_USER=
+DBT_ENV_SECRET_REDSHIFT_PASS=
+REDSHIFT_DATABASE=
+REDSHIFT_PORT=
+REDSHIFT_SCHEMA=dbt_utils_integration_tests_redshift
diff --git a/integration_tests/.env/snowflake.env b/integration_tests/.env/snowflake.env
@@ -1,6 +1,7 @@
-SNOWFLAKE_TEST_ACCOUNT=
-SNOWFLAKE_TEST_USER=
-SNOWFLAKE_TEST_PASSWORD=
-SNOWFLAKE_TEST_ROLE=
-SNOWFLAKE_TEST_DATABASE=
-SNOWFLAKE_TEST_WAREHOUSE=
+SNOWFLAKE_ACCOUNT=
+SNOWFLAKE_USER=
+DBT_ENV_SECRET_SNOWFLAKE_PASS=
+SNOWFLAKE_ROLE=
+SNOWFLAKE_DATABASE=
+SNOWFLAKE_WAREHOUSE=
+SNOWFLAKE_SCHEMA=dbt_utils_integration_tests_snowflake
diff --git a/integration_tests/README.md b/integration_tests/README.md
@@ -13,7 +13,7 @@
 - Docker
 
 ### Configure credentials
-Edit the env file for your TARGET in `integration_tests/.env/[TARGET].env`.
+Edit the env file for your TARGET in `integration_tests/.env/[TARGET].env`.  These will be used for your profiles.yml.
 
 Load the environment variables:
 ```shell
@@ -91,16 +91,6 @@ Where possible, targets are being run in docker containers (this works for Postg
 
 ### Creating a new integration test
 
-#### Set up profiles
-Do either one of the following:
-1. Use `DBT_PROFILES_DIR`
-    ```shell
-    cp integration_tests/ci/sample.profiles.yml integration_tests/profiles.yml
-    export DBT_PROFILES_DIR=$(cd integration_tests && pwd)
-    ```
-2. Use `~/.dbt/profiles.yml`
-    - Copy contents from `integration_tests/ci/sample.profiles.yml` into `~/.dbt/profiles.yml`.
-
 #### Add your integration test
 This directory contains an example dbt project which tests the macros in the `dbt-utils` package. An integration test typically involves making 1) a new seed file 2) a new model file 3) a generic test to assert anticipated behaviour.
 

diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
diff --git a/integration_tests/data/schema_tests/data_test_at_least_one.csv b/integration_tests/data/schema_tests/data_test_at_least_one.csv
@@ -1,2 +1,3 @@
-field
-a
+field,value
+a,1
+b,
diff --git a/integration_tests/data/sql/data_unpivot_quote.csv b/integration_tests/data/sql/data_unpivot_quote.csv
@@ -0,0 +1,4 @@
+Customer_Id,Created_At,sTaTuS,SEGMENT,Name
+123,2017-01-01,active,tier 1,name 1
+234,2017-02-01,active,tier 3,name 3
+567,2017-03-01,churned,tier 2,name 2
diff --git a/integration_tests/data/sql/data_unpivot_quote_expected.csv b/integration_tests/data/sql/data_unpivot_quote_expected.csv
@@ -0,0 +1,7 @@
+Customer_Id,Created_At,Prop,Val
+123,"2017-01-01","SEGMENT","tier 1"
+123,"2017-01-01","sTaTuS","active"
+234,"2017-02-01","SEGMENT","tier 3"
+234,"2017-02-01","sTaTuS","active"
+567,"2017-03-01","sTaTuS","churned"
+567,"2017-03-01","SEGMENT","tier 2"
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -52,6 +52,12 @@ seeds:
           num_buckets: integer
           min_value: float
           max_value: float
+
+      data_unpivot_quote:
+        +quote_columns: true
+
+      data_unpivot_quote_expected:
+        +quote_columns: true
 
     schema_tests:
       data_test_sequential_timestamps:

diff --git a/integration_tests/models/generic_tests/schema.yml b/integration_tests/models/generic_tests/schema.yml
@@ -14,8 +14,12 @@ seeds:
       - name: field
         data_tests:
           - dbt_utils.at_least_one
+      - name: value
+        data_tests:
           - dbt_utils.at_least_one:
               group_by_columns: ['field']
+              error_if: "<1"
+              warn_if: "<0"
 
   - name: data_test_expression_is_true
     data_tests:

diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml
@@ -158,6 +158,11 @@ models:
       - dbt_utils.equality:
           compare_model: ref('data_unpivot_bool_expected')
 
+  - name: test_unpivot_quote
+    data_tests:
+      - dbt_utils.equality:
+          compare_model: ref('data_unpivot_quote_expected')
+
   - name: test_star
     data_tests:
       - dbt_utils.equality:

diff --git a/integration_tests/models/sql/test_unpivot_quote.sql b/integration_tests/models/sql/test_unpivot_quote.sql
@@ -0,0 +1,10 @@
+
+    {{ dbt_utils.unpivot(
+        relation=ref('data_unpivot_quote'),
+        cast_to=type_string(),
+        exclude=['Customer_Id', 'Created_At'],
+        remove=['Name'],
+        field_name='Prop',
+        value_name='Val',
+        quote_identifiers=True,
+    ) }}