From a1711447db3bf948641b8f45d4f0f7e320a3e1a9 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Jan 2024 11:54:02 +0100 Subject: [PATCH 01/14] move all secrets to google vault --- .github/workflows/test_dbt_runner.yml | 10 ---------- .github/workflows/test_destination_athena.yml | 7 ------- .../test_destination_athena_iceberg.yml | 7 ------- .../workflows/test_destination_bigquery.yml | 9 +-------- .github/workflows/test_destination_mssql.yml | 3 --- .github/workflows/test_destination_qdrant.yml | 2 -- .../workflows/test_destination_snowflake.yml | 11 ----------- .github/workflows/test_destination_synapse.yml | 2 -- .../workflows/test_destination_weaviate.yml | 4 ---- .github/workflows/test_destinations.yml | 18 ------------------ .github/workflows/test_doc_snippets.yml | 6 ------ .github/workflows/test_local_destinations.yml | 2 -- 12 files changed, 1 insertion(+), 80 deletions(-) diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml index 1803a53fc1..ab0c47dc2c 100644 --- a/.github/workflows/test_dbt_runner.yml +++ b/.github/workflows/test_dbt_runner.yml @@ -9,16 +9,6 @@ on: workflow_dispatch: env: - # all credentials must be present to be passed to dbt runner - DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader@dlttests.cwz0jfxu0m7m.eu-central-1.rds.amazonaws.com:5432/dlt_data - DESTINATION__REDSHIFT__CREDENTIALS: postgresql://loader@3.73.90.3:5439/dlt_ci - DESTINATION__SNOWFLAKE__CREDENTIALS: snowflake://loader@kgiotue-wn98412/dlt_data?warehouse=COMPUTE_WH&role=DLT_LOADER_ROLE - DESTINATION__CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }} - - DESTINATION__CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci - DESTINATION__CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com - DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }} - DESTINATION__CREDENTIALS__TOKEN_URI: https://oauth2.googleapis.com/token RUNTIME__LOG_LEVEL: ERROR diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index 2d57f26a51..07da036676 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -9,13 +9,6 @@ on: workflow_dispatch: env: - DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK - DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - DESTINATION__ATHENA__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK - DESTINATION__ATHENA__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - DESTINATION__ATHENA__CREDENTIALS__REGION_NAME: eu-central-1 - DESTINATION__ATHENA__QUERY_RESULT_BUCKET: s3://dlt-athena-output - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index d8d8521063..2a7aeb8fc1 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -9,13 +9,6 @@ on: workflow_dispatch: env: - DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK - DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - DESTINATION__ATHENA__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK - DESTINATION__ATHENA__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - DESTINATION__ATHENA__CREDENTIALS__REGION_NAME: eu-central-1 - DESTINATION__ATHENA__QUERY_RESULT_BUCKET: s3://dlt-athena-output - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index 45dc78a085..341cbe5e46 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -9,18 +9,11 @@ on: workflow_dispatch: env: - CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci - CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com - CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }} - CREDENTIALS__TOKEN_URI: https://oauth2.googleapis.com/token + # TODO: migrate to google secrets CREDENTIALS__CLIENT_ID: 929384042504-3mtjaj1s7vuvf53j88mgdq4te7akkjm3.apps.googleusercontent.com CREDENTIALS__CLIENT_SECRET: ${{ secrets.CREDENTIALS__CLIENT_SECRET }} CREDENTIALS__REFRESH_TOKEN: ${{ secrets.CREDENTIALS__REFRESH_TOKEN }} - # needed for bigquery staging tests - # DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK - # DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index 6eb4427bbf..ab3b2f5915 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -9,9 +9,6 @@ on: workflow_dispatch: env: - DESTINATION__MSSQL__CREDENTIALS: mssql://dlt_root@dlt-ci-mssql.database.windows.net:1433/dlt_ci - DESTINATION__MSSQL__CREDENTIALS__PASSWORD: ${{ secrets.MSSQL_PASSWORD }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index 0ce3e3a3f9..0cb0ed42f5 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -8,8 +8,6 @@ on: workflow_dispatch: env: - DESTINATION__QDRANT__CREDENTIALS__LOCATION: ${{ secrets.DESTINATION__QDRANT__CREDENTIALS__LOCATION }} - DESTINATION__QDRANT__CREDENTIALS__API_KEY: ${{ secrets.DESTINATION__QDRANT__CREDENTIALS__API_KEY }} RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index afc4263daf..8fb80c6c29 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -9,17 +9,6 @@ on: workflow_dispatch: env: - DESTINATION__SNOWFLAKE__CREDENTIALS: snowflake://loader@kgiotue-wn98412/dlt_data?warehouse=COMPUTE_WH&role=DLT_LOADER_ROLE - CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }} - - # needed for snowflake staging tests - DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK - DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - DESTINATION__FILESYSTEM__CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci - DESTINATION__FILESYSTEM__CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com - DESTINATION__FILESYSTEM__CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }} - DESTINATION__FILESYSTEM__CREDENTIALS__AZURE_STORAGE_ACCOUNT_NAME: dltdata - DESTINATION__FILESYSTEM__CREDENTIALS__AZURE_STORAGE_ACCOUNT_KEY: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }} RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml index d0f364c382..3e694d847a 100644 --- a/.github/workflows/test_destination_synapse.yml +++ b/.github/workflows/test_destination_synapse.yml @@ -9,8 +9,6 @@ on: workflow_dispatch: env: - DESTINATION__SYNAPSE__CREDENTIALS: ${{ secrets.SYNAPSE_CREDENTIALS }} - DESTINATION__SYNAPSE__CREDENTIALS__PASSWORD: ${{ secrets.SYNAPSE_PASSWORD }} RUNTIME__SENTRY_DSN: https://cf6086f7d263462088b9fb9f9947caee@o4505514867163136.ingest.sentry.io/4505516212682752 RUNTIME__LOG_LEVEL: ERROR diff --git a/.github/workflows/test_destination_weaviate.yml b/.github/workflows/test_destination_weaviate.yml index c771a28204..1f242f8b2e 100644 --- a/.github/workflows/test_destination_weaviate.yml +++ b/.github/workflows/test_destination_weaviate.yml @@ -8,10 +8,6 @@ on: workflow_dispatch: env: - DESTINATION__WEAVIATE__CREDENTIALS__URL: ${{ secrets.DESTINATION__WEAVIATE__CREDENTIALS__URL }} - DESTINATION__WEAVIATE__CREDENTIALS__API_KEY: ${{ secrets.DESTINATION__WEAVIATE__CREDENTIALS__API_KEY }} - DESTINATION__WEAVIATE__CREDENTIALS__ADDITIONAL_HEADERS: ${{ secrets.DESTINATION__WEAVIATE__CREDENTIALS__ADDITIONAL_HEADERS }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index 6aaf440dcd..8787a8986b 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -9,30 +9,12 @@ on: workflow_dispatch: env: - DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader@dlttests.cwz0jfxu0m7m.eu-central-1.rds.amazonaws.com:5432/dlt_data - DESTINATION__DUCKDB__CREDENTIALS: duckdb:///_storage/test_quack.duckdb - DESTINATION__REDSHIFT__CREDENTIALS: postgresql://loader@3.73.90.3:5439/dlt_ci - DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK - DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - DESTINATION__FILESYSTEM__CREDENTIALS__AZURE_STORAGE_ACCOUNT_NAME: dltdata - DESTINATION__FILESYSTEM__CREDENTIALS__AZURE_STORAGE_ACCOUNT_KEY: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }} # For s3 compatible tests TESTS__R2_AWS_ACCESS_KEY_ID: a4950a5003b26f5a71ac97ef3848ff4c TESTS__R2_AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} TESTS__R2_ENDPOINT_URL: https://9830548e4e4b582989be0811f2a0a97f.r2.cloudflarestorage.com - # DESTINATION__ATHENA__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK - # DESTINATION__ATHENA__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # DESTINATION__ATHENA__CREDENTIALS__REGION_NAME: eu-central-1 - # DESTINATION__ATHENA__QUERY_RESULT_BUCKET: s3://dlt-athena-output - - # password is the same so it will be shared - CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci - CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com - CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }} - CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB diff --git a/.github/workflows/test_doc_snippets.yml b/.github/workflows/test_doc_snippets.yml index 29e7db0c4d..ecab8aeb62 100644 --- a/.github/workflows/test_doc_snippets.yml +++ b/.github/workflows/test_doc_snippets.yml @@ -11,8 +11,6 @@ on: env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - DESTINATION__DUCKDB__CREDENTIALS: duckdb:///_storage/test_quack.duckdb - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB @@ -23,10 +21,6 @@ env: # Slack hook for chess in production example RUNTIME__SLACK_INCOMING_HOOK: ${{ secrets.RUNTIME__SLACK_INCOMING_HOOK }} - # Qdrant credentials - DESTINATION__QDRANT__CREDENTIALS__LOCATION: ${{ secrets.DESTINATION__QDRANT__CREDENTIALS__LOCATION }} - DESTINATION__QDRANT__CREDENTIALS__API_KEY: ${{ secrets.DESTINATION__QDRANT__CREDENTIALS__API_KEY }} - jobs: run_lint: diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index 42c3c2d13a..4ddebbda3b 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -11,8 +11,6 @@ on: workflow_dispatch: env: - DESTINATION__DUCKDB__CREDENTIALS: duckdb:///_storage/test_quack.duckdb - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB From 78494b36b45c1feab18e2b865b6c94f4f260ef6e Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Jan 2024 12:03:33 +0100 Subject: [PATCH 02/14] put dbt env vars back in --- .github/workflows/test_dbt_runner.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml index ab0c47dc2c..1803a53fc1 100644 --- a/.github/workflows/test_dbt_runner.yml +++ b/.github/workflows/test_dbt_runner.yml @@ -9,6 +9,16 @@ on: workflow_dispatch: env: + # all credentials must be present to be passed to dbt runner + DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader@dlttests.cwz0jfxu0m7m.eu-central-1.rds.amazonaws.com:5432/dlt_data + DESTINATION__REDSHIFT__CREDENTIALS: postgresql://loader@3.73.90.3:5439/dlt_ci + DESTINATION__SNOWFLAKE__CREDENTIALS: snowflake://loader@kgiotue-wn98412/dlt_data?warehouse=COMPUTE_WH&role=DLT_LOADER_ROLE + DESTINATION__CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }} + + DESTINATION__CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci + DESTINATION__CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com + DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }} + DESTINATION__CREDENTIALS__TOKEN_URI: https://oauth2.googleapis.com/token RUNTIME__LOG_LEVEL: ERROR From 6092a61a7a5089c1fcad09dbc5635c047ec72bdd Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Jan 2024 13:04:45 +0100 Subject: [PATCH 03/14] force rerun of tests --- .github/workflows/test_destination_bigquery.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index 341cbe5e46..e6e05d8526 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -14,6 +14,7 @@ env: CREDENTIALS__CLIENT_SECRET: ${{ secrets.CREDENTIALS__CLIENT_SECRET }} CREDENTIALS__REFRESH_TOKEN: ${{ secrets.CREDENTIALS__REFRESH_TOKEN }} + RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR From c7d2e42fd275d1eff06991ee7717bb79b5cb4b45 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Jan 2024 13:13:20 +0100 Subject: [PATCH 04/14] add missing secrets injection --- .github/workflows/test_dbt_runner.yml | 6 ++++++ .github/workflows/test_destination_athena.yml | 5 +++++ .github/workflows/test_destination_athena_iceberg.yml | 5 +++++ .github/workflows/test_destination_bigquery.yml | 7 ++++--- .github/workflows/test_destination_mssql.yml | 6 ++++++ .github/workflows/test_destination_qdrant.yml | 4 ++++ .github/workflows/test_destination_snowflake.yml | 4 ++++ .github/workflows/test_destination_synapse.yml | 4 ++++ .github/workflows/test_destination_weaviate.yml | 5 +++++ .github/workflows/test_destinations.yml | 6 ++++-- .github/workflows/test_local_destinations.yml | 5 +++++ 11 files changed, 52 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml index 1803a53fc1..979f409556 100644 --- a/.github/workflows/test_dbt_runner.yml +++ b/.github/workflows/test_dbt_runner.yml @@ -9,6 +9,9 @@ on: workflow_dispatch: env: + + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + # all credentials must be present to be passed to dbt runner DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader@dlttests.cwz0jfxu0m7m.eu-central-1.rds.amazonaws.com:5432/dlt_data DESTINATION__REDSHIFT__CREDENTIALS: postgresql://loader@3.73.90.3:5439/dlt_ci @@ -84,6 +87,9 @@ jobs: # install dlt with postgres support run: poetry run pip uninstall dbt-core -y + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: | poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -k '(not local)' if: runner.os != 'Windows' diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index 07da036676..e9e17edefe 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -9,6 +9,8 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB @@ -65,6 +67,9 @@ jobs: # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction -E athena --with sentry-sdk --with pipeline + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: | poetry run pytest tests/load if: runner.os != 'Windows' diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index 2a7aeb8fc1..92b73d5a9b 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -9,6 +9,8 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB @@ -65,6 +67,9 @@ jobs: # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction -E --with sentry-sdk --with pipeline + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: | poetry run pytest tests/load if: runner.os != 'Windows' diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index e6e05d8526..fa32977745 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -9,12 +9,13 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + # TODO: migrate to google secrets CREDENTIALS__CLIENT_ID: 929384042504-3mtjaj1s7vuvf53j88mgdq4te7akkjm3.apps.googleusercontent.com CREDENTIALS__CLIENT_SECRET: ${{ secrets.CREDENTIALS__CLIENT_SECRET }} CREDENTIALS__REFRESH_TOKEN: ${{ secrets.CREDENTIALS__REFRESH_TOKEN }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR @@ -75,8 +76,8 @@ jobs: # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction -E bigquery --with providers -E parquet --with sentry-sdk --with pipeline - # - name: Install self - # run: poetry install --no-interaction + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | poetry run pytest tests/helpers/providers tests/load diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index ab3b2f5915..b8ea1db2d4 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -9,6 +9,9 @@ on: workflow_dispatch: env: + + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR @@ -64,6 +67,9 @@ jobs: - name: Install dependencies run: poetry install --no-interaction -E mssql -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: | poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py if: runner.os != 'Windows' diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index 0cb0ed42f5..758c18b56b 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -8,6 +8,7 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR @@ -56,6 +57,9 @@ jobs: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - name: Install dependencies run: poetry install --no-interaction -E qdrant -E parquet --with sentry-sdk --with pipeline - run: | diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index 8fb80c6c29..979ea3e917 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -9,6 +9,7 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR @@ -62,6 +63,9 @@ jobs: - name: Install dependencies run: poetry install --no-interaction -E snowflake -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: | poetry run pytest tests/load if: runner.os != 'Windows' diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml index 3e694d847a..83800fa789 100644 --- a/.github/workflows/test_destination_synapse.yml +++ b/.github/workflows/test_destination_synapse.yml @@ -9,6 +9,7 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} RUNTIME__SENTRY_DSN: https://cf6086f7d263462088b9fb9f9947caee@o4505514867163136.ingest.sentry.io/4505516212682752 RUNTIME__LOG_LEVEL: ERROR @@ -70,6 +71,9 @@ jobs: - name: Install dependencies run: poetry install --no-interaction -E synapse -E s3 -E gs -E az --with sentry-sdk --with pipeline + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: | poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py if: runner.os != 'Windows' diff --git a/.github/workflows/test_destination_weaviate.yml b/.github/workflows/test_destination_weaviate.yml index 1f242f8b2e..8c0161f364 100644 --- a/.github/workflows/test_destination_weaviate.yml +++ b/.github/workflows/test_destination_weaviate.yml @@ -8,6 +8,8 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR @@ -56,6 +58,9 @@ jobs: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - name: Install dependencies run: poetry install --no-interaction -E weaviate -E parquet --with sentry-sdk --with pipeline - run: | diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index 8787a8986b..c60d870b05 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -10,6 +10,8 @@ on: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + # For s3 compatible tests TESTS__R2_AWS_ACCESS_KEY_ID: a4950a5003b26f5a71ac97ef3848ff4c TESTS__R2_AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} @@ -71,8 +73,8 @@ jobs: # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction -E redshift -E gs -E s3 -E az -E parquet -E duckdb -E cli --with sentry-sdk --with pipeline - # - name: Install self - # run: poetry install --no-interaction + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | poetry run pytest tests/load diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index 4ddebbda3b..12f5c0ca48 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -11,6 +11,8 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB @@ -84,6 +86,9 @@ jobs: - name: Install dependencies run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate --with sentry-sdk --with pipeline + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: poetry run pytest tests/load && poetry run pytest tests/cli name: Run tests Linux env: From 44661616cae098d71ac89607c23b417efe0e337f Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Jan 2024 13:29:56 +0100 Subject: [PATCH 05/14] move google api client to dev dependencies --- poetry.lock | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0d1412afd8..91d01a29b7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -8317,4 +8317,4 @@ weaviate = ["weaviate-client"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.13" -content-hash = "e1685f6e1f0f3032b21e327b0523214a13d1f805c367bb7a8a3ca6cc6643e847" +content-hash = "c6dbf511fe1652e0c7a88ccd620805e35ef0bc6e84e672b6e49a800d2d98c3f5" diff --git a/pyproject.toml b/pyproject.toml index f6f97b431b..8af0f7911f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -132,6 +132,7 @@ boto3-stubs = "^1.28.28" types-tqdm = "^4.66.0.2" types-psutil = "^5.9.5.16" types-psycopg2 = "^2.9.21.14" +google-api-python-client = ">=1.7.11" [tool.poetry.group.pipeline] optional=true @@ -174,7 +175,6 @@ dbt-core=">=1.2.0" dbt-duckdb=">=1.2.0" pymongo = ">=4.3.3" pandas = ">2" -google-api-python-client = ">=1.7.11" [tool.black] # https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html#configuration-via-a-file line-length = 100 From 5e36753b5e45462fd4dfb4f738c004a6f219a28f Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Jan 2024 13:36:29 +0100 Subject: [PATCH 06/14] move more weaviate creds --- .github/workflows/test_doc_snippets.yml | 3 --- .github/workflows/test_local_destinations.yml | 3 --- 2 files changed, 6 deletions(-) diff --git a/.github/workflows/test_doc_snippets.yml b/.github/workflows/test_doc_snippets.yml index ecab8aeb62..80b77ce6c9 100644 --- a/.github/workflows/test_doc_snippets.yml +++ b/.github/workflows/test_doc_snippets.yml @@ -15,9 +15,6 @@ env: RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB - DESTINATION__WEAVIATE__VECTORIZER: text2vec-contextionary - DESTINATION__WEAVIATE__MODULE_CONFIG: "{\"text2vec-contextionary\": {\"vectorizeClassName\": false, \"vectorizePropertyName\": true}}" - # Slack hook for chess in production example RUNTIME__SLACK_INCOMING_HOOK: ${{ secrets.RUNTIME__SLACK_INCOMING_HOOK }} diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index 12f5c0ca48..a02957b69d 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -19,9 +19,6 @@ env: ACTIVE_DESTINATIONS: "[\"duckdb\", \"postgres\", \"filesystem\", \"weaviate\"]" ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\"]" - DESTINATION__WEAVIATE__VECTORIZER: text2vec-contextionary - DESTINATION__WEAVIATE__MODULE_CONFIG: "{\"text2vec-contextionary\": {\"vectorizeClassName\": false, \"vectorizePropertyName\": true}}" - jobs: get_docs_changes: uses: ./.github/workflows/get_docs_changes.yml From 1ad117fe4d6b8c8b6a5d573fe998fcc0db3221e4 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Jan 2024 13:49:02 +0100 Subject: [PATCH 07/14] force rerun --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index c1cb9bec98..bd425b0e42 100644 --- a/Makefile +++ b/Makefile @@ -94,3 +94,4 @@ test-build-images: build-library docker build -f deploy/dlt/Dockerfile.airflow --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" . docker build -f deploy/dlt/Dockerfile --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" . + From d9737dca744e03eaf1b7d6f85bcecec52e8ee10a Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Jan 2024 15:22:50 +0100 Subject: [PATCH 08/14] remove weaviate test workflow --- .../workflows/test_destination_weaviate.yml | 85 ------------------- 1 file changed, 85 deletions(-) delete mode 100644 .github/workflows/test_destination_weaviate.yml diff --git a/.github/workflows/test_destination_weaviate.yml b/.github/workflows/test_destination_weaviate.yml deleted file mode 100644 index 8c0161f364..0000000000 --- a/.github/workflows/test_destination_weaviate.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: test weaviate - -on: - pull_request: - branches: - - master - - devel - workflow_dispatch: - -env: - DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 - RUNTIME__LOG_LEVEL: ERROR - - ACTIVE_DESTINATIONS: "[\"weaviate\"]" - ALL_FILESYSTEM_DRIVERS: "[\"memory\"]" - -jobs: - get_docs_changes: - uses: ./.github/workflows/get_docs_changes.yml - if: ${{ !github.event.pull_request.head.repo.fork }} - - run_loader: - name: Tests Weaviate loader - needs: get_docs_changes - if: !always() - # if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest", "macos-latest", "windows-latest"] - defaults: - run: - shell: bash - runs-on: ${{ matrix.os }} - - steps: - - name: Check out - uses: actions/checkout@master - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: "3.10.x" - - - name: Install Poetry - uses: snok/install-poetry@v1.3.2 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v3 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp - - - name: create secrets.toml - run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - - - name: Install dependencies - run: poetry install --no-interaction -E weaviate -E parquet --with sentry-sdk --with pipeline - - run: | - poetry run pytest tests/load/ - if: runner.os != 'Windows' - name: Run tests Linux/MAC - - run: | - poetry run pytest tests/load/ - if: runner.os == 'Windows' - name: Run tests Windows - shell: cmd - - matrix_job_required_check: - name: Weaviate loader tests - needs: run_loader - runs-on: ubuntu-latest - if: always() - steps: - - name: Check matrix job results - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1 From 0160215758af28c7a5a12c2155fc0fa7ef3123fe Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Jan 2024 15:45:38 +0100 Subject: [PATCH 09/14] move bigquery oauth secrets to google secrets --- .github/workflows/test_destination_bigquery.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index fa32977745..66eb3fe100 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -11,11 +11,6 @@ on: env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - # TODO: migrate to google secrets - CREDENTIALS__CLIENT_ID: 929384042504-3mtjaj1s7vuvf53j88mgdq4te7akkjm3.apps.googleusercontent.com - CREDENTIALS__CLIENT_SECRET: ${{ secrets.CREDENTIALS__CLIENT_SECRET }} - CREDENTIALS__REFRESH_TOKEN: ${{ secrets.CREDENTIALS__REFRESH_TOKEN }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR From b07d873fd24d93dd98e2d2fb71d9c46c12df3e8d Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Jan 2024 16:10:34 +0100 Subject: [PATCH 10/14] fix destination config resolution test --- tests/load/pipeline/test_pipelines.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/load/pipeline/test_pipelines.py b/tests/load/pipeline/test_pipelines.py index abbb2b022f..d170fd553b 100644 --- a/tests/load/pipeline/test_pipelines.py +++ b/tests/load/pipeline/test_pipelines.py @@ -478,26 +478,29 @@ def test_pipeline_explicit_destination_credentials( ) -> None: # explicit credentials resolved p = dlt.pipeline( - destination="postgres", credentials="postgresql://loader:loader@localhost:5432/dlt_data" + destination=Destination.from_reference("postgres", destination_name="mydest"), + credentials="postgresql://loader:loader@localhost:7777/dlt_data", ) c = p._get_destination_clients(Schema("s"), p._get_destination_client_initial_config())[0] - assert c.config.credentials.host == "localhost" # type: ignore[attr-defined] + assert c.config.credentials.port == 7777 # type: ignore[attr-defined] - # Remove connection string in CI to start with clean environ # TODO: may want to clear the env completely and ignore/mock config files somehow to avoid side effects - os.environ.pop("DESTINATION__POSTGRES__CREDENTIALS", None) # explicit credentials resolved ignoring the config providers - os.environ["DESTINATION__POSTGRES__CREDENTIALS__HOST"] = "HOST" + os.environ["DESTINATION__MYDEST__CREDENTIALS__HOST"] = "HOST" p = dlt.pipeline( - destination="postgres", credentials="postgresql://loader:loader@localhost:5432/dlt_data" + destination=Destination.from_reference("postgres", destination_name="mydest"), + credentials="postgresql://loader:loader@localhost:5432/dlt_data", ) c = p._get_destination_clients(Schema("s"), p._get_destination_client_initial_config())[0] assert c.config.credentials.host == "localhost" # type: ignore[attr-defined] # explicit partial credentials will use config providers - os.environ["DESTINATION__POSTGRES__CREDENTIALS__USERNAME"] = "UN" - os.environ["DESTINATION__POSTGRES__CREDENTIALS__PASSWORD"] = "PW" - p = dlt.pipeline(destination="postgres", credentials="postgresql://localhost:5432/dlt_data") + os.environ["DESTINATION__MYDEST__CREDENTIALS__USERNAME"] = "UN" + os.environ["DESTINATION__MYDEST__CREDENTIALS__PASSWORD"] = "PW" + p = dlt.pipeline( + destination=Destination.from_reference("postgres", destination_name="mydest"), + credentials="postgresql://localhost:5432/dlt_data", + ) c = p._get_destination_clients(Schema("s"), p._get_destination_client_initial_config())[0] assert c.config.credentials.username == "UN" # type: ignore[attr-defined] # host is also overridden From 969ac23cc3dbb799e1e7e66ad9bc5d9a0b509989 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Jan 2024 17:22:36 +0100 Subject: [PATCH 11/14] disable google api client logging for cli tests --- tests/cli/test_pipeline_command.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/cli/test_pipeline_command.py b/tests/cli/test_pipeline_command.py index 168b172d07..1f8e2ff4f3 100644 --- a/tests/cli/test_pipeline_command.py +++ b/tests/cli/test_pipeline_command.py @@ -2,6 +2,7 @@ import os import contextlib import pytest +import logging from subprocess import CalledProcessError import dlt @@ -177,6 +178,9 @@ def test_pipeline_command_failed_jobs(repo_dir: str, project_files: FileStorage) print(cpe.stderr) raise + # disable logging output for discovery cache for this test + logging.getLogger("googleapiclient.discovery_cache").setLevel(logging.ERROR) + with io.StringIO() as buf, contextlib.redirect_stdout(buf): pipeline_command.pipeline_command("trace", "chess_pipeline", None, 0) _out = buf.getvalue() From dfa184418a1ee26388d17e917d4098cb999635cd Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Jan 2024 21:28:02 +0100 Subject: [PATCH 12/14] fix redshift conf test --- tests/load/redshift/test_redshift_table_builder.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/load/redshift/test_redshift_table_builder.py b/tests/load/redshift/test_redshift_table_builder.py index d2adfde403..4ad66b6f6b 100644 --- a/tests/load/redshift/test_redshift_table_builder.py +++ b/tests/load/redshift/test_redshift_table_builder.py @@ -35,11 +35,13 @@ def test_redshift_configuration() -> None: # check names normalized with custom_environ( { - "DESTINATION__REDSHIFT__CREDENTIALS__DATABASE": "UPPER_CASE_DATABASE", - "DESTINATION__REDSHIFT__CREDENTIALS__PASSWORD": " pass\n", + "DESTINATION__MY_REDSHIFT__CREDENTIALS__USERNAME": "username", + "DESTINATION__MY_REDSHIFT__CREDENTIALS__HOST": "host", + "DESTINATION__MY_REDSHIFT__CREDENTIALS__DATABASE": "UPPER_CASE_DATABASE", + "DESTINATION__MY_REDSHIFT__CREDENTIALS__PASSWORD": " pass\n", } ): - C = resolve_configuration(RedshiftCredentials(), sections=("destination", "redshift")) + C = resolve_configuration(RedshiftCredentials(), sections=("destination", "my_redshift")) assert C.database == "upper_case_database" assert C.password == "pass" From 4bb534ab3a2aff5e8f510e8dbe130679e782141a Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Jan 2024 21:28:34 +0100 Subject: [PATCH 13/14] check dbt runner needs env vars --- .github/workflows/test_dbt_runner.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml index 979f409556..5ae791c979 100644 --- a/.github/workflows/test_dbt_runner.yml +++ b/.github/workflows/test_dbt_runner.yml @@ -11,18 +11,6 @@ on: env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - - # all credentials must be present to be passed to dbt runner - DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader@dlttests.cwz0jfxu0m7m.eu-central-1.rds.amazonaws.com:5432/dlt_data - DESTINATION__REDSHIFT__CREDENTIALS: postgresql://loader@3.73.90.3:5439/dlt_ci - DESTINATION__SNOWFLAKE__CREDENTIALS: snowflake://loader@kgiotue-wn98412/dlt_data?warehouse=COMPUTE_WH&role=DLT_LOADER_ROLE - DESTINATION__CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }} - - DESTINATION__CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci - DESTINATION__CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com - DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }} - DESTINATION__CREDENTIALS__TOKEN_URI: https://oauth2.googleapis.com/token - RUNTIME__LOG_LEVEL: ERROR jobs: @@ -73,6 +61,9 @@ jobs: # install dlt with postgres support run: poetry install --no-interaction -E postgres -E dbt --with sentry-sdk + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: | poetry run pytest tests/helpers/dbt_tests -k '(not venv)' if: runner.os != 'Windows' @@ -87,9 +78,6 @@ jobs: # install dlt with postgres support run: poetry run pip uninstall dbt-core -y - - name: create secrets.toml - run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - - run: | poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -k '(not local)' if: runner.os != 'Windows' From f915f87439efdcf1d8d19af0dc6c427bebd8d508 Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 5 Jan 2024 08:52:28 +0100 Subject: [PATCH 14/14] fix bq tests --- tests/load/bigquery/test_bigquery_client.py | 4 +++- tests/load/bigquery/test_bigquery_table_builder.py | 13 +++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/load/bigquery/test_bigquery_client.py b/tests/load/bigquery/test_bigquery_client.py index a2ca9c53b1..2e7930c339 100644 --- a/tests/load/bigquery/test_bigquery_client.py +++ b/tests/load/bigquery/test_bigquery_client.py @@ -182,7 +182,9 @@ def _assert_credentials(gcp_credentials): def test_get_oauth_access_token() -> None: - c = resolve_configuration(GcpOAuthCredentialsWithoutDefaults()) + c = resolve_configuration( + GcpOAuthCredentialsWithoutDefaults(), sections=("destination", "bigquery") + ) assert c.refresh_token is not None assert c.token is None c.auth() diff --git a/tests/load/bigquery/test_bigquery_table_builder.py b/tests/load/bigquery/test_bigquery_table_builder.py index d622f9205c..4f40524196 100644 --- a/tests/load/bigquery/test_bigquery_table_builder.py +++ b/tests/load/bigquery/test_bigquery_table_builder.py @@ -1,3 +1,4 @@ +import os import pytest import sqlfluff from copy import deepcopy @@ -21,13 +22,17 @@ def schema() -> Schema: def test_configuration() -> None: + os.environ["MYBG__CREDENTIALS__CLIENT_EMAIL"] = "1234" + os.environ["MYBG__CREDENTIALS__PRIVATE_KEY"] = "1234" + os.environ["MYBG__CREDENTIALS__PROJECT_ID"] = "1234" + # check names normalized - with custom_environ({"CREDENTIALS__PRIVATE_KEY": "---NO NEWLINE---\n"}): - C = resolve_configuration(GcpServiceAccountCredentialsWithoutDefaults()) + with custom_environ({"MYBG__CREDENTIALS__PRIVATE_KEY": "---NO NEWLINE---\n"}): + C = resolve_configuration(GcpServiceAccountCredentialsWithoutDefaults(), sections=("mybg",)) assert C.private_key == "---NO NEWLINE---\n" - with custom_environ({"CREDENTIALS__PRIVATE_KEY": "---WITH NEWLINE---\n"}): - C = resolve_configuration(GcpServiceAccountCredentialsWithoutDefaults()) + with custom_environ({"MYBG__CREDENTIALS__PRIVATE_KEY": "---WITH NEWLINE---\n"}): + C = resolve_configuration(GcpServiceAccountCredentialsWithoutDefaults(), sections=("mybg",)) assert C.private_key == "---WITH NEWLINE---\n"