updates to get BQ working

dbt-labs · Jul 24, 2024 · 7408dc7 · 7408dc7
1 parent c3bbe0b
commit 7408dc7
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 14 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -63,7 +63,7 @@ jobs:
                   echo $supported_adapters
                   echo "adapters=$supported_adapters" >> $GITHUB_OUTPUT
 
-            - name: "[ANNOTATION] ${{ github.event.repository.name }} - adapters to test"
+            - name: "[ANNOTATION] ${{ github.event.repository.name }} - Testing ${{ steps.supported-adapters.outputs.adapters }}"
               run: |
                   title="${{ github.event.repository.name }} - adapters to test"
                   message="The workflow will run tests for the following adapters: ${{ steps.supported-adapters.outputs.adapters }}"
@@ -121,22 +121,19 @@ jobs:
                   POSTGRES_PORT: 5432
                   POSTGRES_DATABASE: ${{ vars.POSTGRES_DATABASE }}
                   POSTGRES_SCHEMA: "dbt_utils_integration_tests_postgres_${{ github.run_number }}"
-                  # snowflake env vars
                   SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
                   SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
                   DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }}
                   SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
                   SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
                   SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
                   SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}"
-                  # redshift
                   REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
                   REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
                   DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }}
                   REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }}
                   REDSHIFT_SCHEMA: "dbt_utils_integration_tests_redshift_${{ github.run_number }}"
                   REDSHIFT_PORT: 5439
-                  # bigquery
                   BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
-                  DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON: "${{ secrets.BIGQUERY_SERVICE_ACCOUNT_JSON }}"
+                  BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}
                   BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery_${{ github.run_number }}"
diff --git a/integration_tests/.env/bigquery.env b/integration_tests/.env/bigquery.env
@@ -1,3 +1,3 @@
-DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON=
+BIGQUERY_KEYFILE_JSON=
 BIGQUERY_PROJECT=
 BIGQUERY_SCHEMA=dbt_utils_integration_tests_bigquery
diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml
@@ -7,7 +7,7 @@ integration_tests:
   target: postgres
   outputs:
     postgres:
-      type: postgres
+      type: "postgres"
       host: "{{ env_var('POSTGRES_HOST') }}"
       user: "{{ env_var('POSTGRES_USER') }}"
       pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}"
@@ -17,7 +17,7 @@ integration_tests:
       threads: 5
 
     redshift:
-      type: redshift
+      type: "redshift"
       host: "{{ env_var('REDSHIFT_HOST') }}"
       user: "{{ env_var('REDSHIFT_USER') }}"
       pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}"
@@ -27,16 +27,17 @@ integration_tests:
       threads: 5
 
     bigquery:
-      type: bigquery
-      method: service-account-json
+      type: "bigquery"
+      method: "service-account-json"
       project: "{{ env_var('BIGQUERY_PROJECT') }}"
-      keyfile_json:
-        "{{ env_var('DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON') | as_native }}"
       dataset: "{{ env_var('BIGQUERY_SCHEMA') }}"
       threads: 10
+      keyfile_json:
+        "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native}}"
+      job_retries: 3
 
     snowflake:
-      type: snowflake
+      type: "snowflake"
       account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}"
       user: "{{ env_var('SNOWFLAKE_USER') }}"
       password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}"

diff --git a/tox.ini b/tox.ini
@@ -28,7 +28,7 @@ passenv =
     REDSHIFT_PORT
     # bigquery
     BIGQUERY_PROJECT
-    DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON
+    BIGQUERY_KEYFILE_JSON
     BIGQUERY_SCHEMA
 
 # Uses pytest to run tests, very basic and doesn't require dbt