Add support for bigquery testing in GitHub CI via tox (#203)

* add support for bigquery testing * add missing var in tox file * Temporarily only run CI tests for BigQuery * Prefix the schema for the data source in CI with the name of the target schema * Store artifacts for logs and target directories for BigQuery * Set up environment variable for BigQuery credentials (keyfile for service account JSON) * Set the custom schema in the source definition * Use the target schema * Try to align actual vs. expected when the schema name is variable * Remove extraneous storage of artifacts * Temporarily disable two failing CI tests * Revert "Temporarily disable two failing CI tests" This reverts commit d70d776. --------- Co-authored-by: Doug Beatty <[email protected]>
dbt-labs · Nov 22, 2024 · a88bc12 · a88bc12
1 parent 972aa3f
commit a88bc12
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 5 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -29,6 +29,9 @@ jobs:
         SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
         SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
         SNOWFLAKE_SCHEMA: "integration_tests_snowflake_${{ github.run_number }}"
+        # bigquery
+        BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
+        BIGQUERY_SCHEMA: "integration_tests_bigquery_${{ github.run_number }}"
         # redshift
         REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
         REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
@@ -39,3 +42,4 @@ jobs:
         SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
         DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }}
         DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }}
+        BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}
diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml
@@ -28,10 +28,11 @@ integration_tests:
 
     bigquery:
       type: bigquery
-      method: service-account
-      keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}"
-      project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}"
-      schema: codegen_integration_tests_bigquery
+      method: service-account-json
+      keyfile_json:
+        "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native}}"
+      project: "{{ env_var('BIGQUERY_PROJECT') }}"
+      dataset: "{{ env_var('BIGQUERY_SCHEMA') }}"
       threads: 1
 
     snowflake:

diff --git a/supported_adapters.env b/supported_adapters.env
@@ -1 +1 @@
-SUPPORTED_ADAPTERS=postgres,snowflake,redshift
+SUPPORTED_ADAPTERS=postgres,snowflake,redshift,bigquery
diff --git a/tox.ini b/tox.ini
@@ -19,6 +19,10 @@ passenv =
     SNOWFLAKE_DATABASE
     SNOWFLAKE_WAREHOUSE
     SNOWFLAKE_SCHEMA
+    # bigquery env vars
+    BIGQUERY_PROJECT
+    BIGQUERY_SCHEMA
+    BIGQUERY_KEYFILE_JSON
     # redshift env vars
     REDSHIFT_HOST
     REDSHIFT_USER
@@ -57,6 +61,21 @@ commands =
     dbt --warn-error run --target snowflake
     dbt --warn-error test --target snowflake
 
+# bigquery integration tests for centralized dbt testing
+# run dbt commands directly, assumes dbt is already installed in environment
+[testenv:dbt_integration_bigquery]
+changedir = integration_tests
+allowlist_externals = 
+    dbt
+skip_install = true
+commands =
+    dbt --version
+    dbt --warn-error deps --target bigquery
+    dbt --warn-error run-operation create_source_table --target bigquery
+    dbt --warn-error seed --target bigquery --full-refresh
+    dbt --warn-error run --target bigquery
+    dbt --warn-error test --target bigquery
+
 # redshift integration tests for centralized dbt testing
 # run dbt commands directly, assumes dbt is already installed in environment
 [testenv:dbt_integration_redshift]