From c7d2e42fd275d1eff06991ee7717bb79b5cb4b45 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Jan 2024 13:13:20 +0100 Subject: [PATCH] add missing secrets injection --- .github/workflows/test_dbt_runner.yml | 6 ++++++ .github/workflows/test_destination_athena.yml | 5 +++++ .github/workflows/test_destination_athena_iceberg.yml | 5 +++++ .github/workflows/test_destination_bigquery.yml | 7 ++++--- .github/workflows/test_destination_mssql.yml | 6 ++++++ .github/workflows/test_destination_qdrant.yml | 4 ++++ .github/workflows/test_destination_snowflake.yml | 4 ++++ .github/workflows/test_destination_synapse.yml | 4 ++++ .github/workflows/test_destination_weaviate.yml | 5 +++++ .github/workflows/test_destinations.yml | 6 ++++-- .github/workflows/test_local_destinations.yml | 5 +++++ 11 files changed, 52 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml index 1803a53fc1..979f409556 100644 --- a/.github/workflows/test_dbt_runner.yml +++ b/.github/workflows/test_dbt_runner.yml @@ -9,6 +9,9 @@ on: workflow_dispatch: env: + + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + # all credentials must be present to be passed to dbt runner DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader@dlttests.cwz0jfxu0m7m.eu-central-1.rds.amazonaws.com:5432/dlt_data DESTINATION__REDSHIFT__CREDENTIALS: postgresql://loader@3.73.90.3:5439/dlt_ci @@ -84,6 +87,9 @@ jobs: # install dlt with postgres support run: poetry run pip uninstall dbt-core -y + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: | poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -k '(not local)' if: runner.os != 'Windows' diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index 07da036676..e9e17edefe 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -9,6 +9,8 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB @@ -65,6 +67,9 @@ jobs: # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction -E athena --with sentry-sdk --with pipeline + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: | poetry run pytest tests/load if: runner.os != 'Windows' diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index 2a7aeb8fc1..92b73d5a9b 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -9,6 +9,8 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB @@ -65,6 +67,9 @@ jobs: # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction -E --with sentry-sdk --with pipeline + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: | poetry run pytest tests/load if: runner.os != 'Windows' diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index e6e05d8526..fa32977745 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -9,12 +9,13 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + # TODO: migrate to google secrets CREDENTIALS__CLIENT_ID: 929384042504-3mtjaj1s7vuvf53j88mgdq4te7akkjm3.apps.googleusercontent.com CREDENTIALS__CLIENT_SECRET: ${{ secrets.CREDENTIALS__CLIENT_SECRET }} CREDENTIALS__REFRESH_TOKEN: ${{ secrets.CREDENTIALS__REFRESH_TOKEN }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR @@ -75,8 +76,8 @@ jobs: # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction -E bigquery --with providers -E parquet --with sentry-sdk --with pipeline - # - name: Install self - # run: poetry install --no-interaction + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | poetry run pytest tests/helpers/providers tests/load diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index ab3b2f5915..b8ea1db2d4 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -9,6 +9,9 @@ on: workflow_dispatch: env: + + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR @@ -64,6 +67,9 @@ jobs: - name: Install dependencies run: poetry install --no-interaction -E mssql -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: | poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py if: runner.os != 'Windows' diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index 0cb0ed42f5..758c18b56b 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -8,6 +8,7 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR @@ -56,6 +57,9 @@ jobs: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - name: Install dependencies run: poetry install --no-interaction -E qdrant -E parquet --with sentry-sdk --with pipeline - run: | diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index 8fb80c6c29..979ea3e917 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -9,6 +9,7 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR @@ -62,6 +63,9 @@ jobs: - name: Install dependencies run: poetry install --no-interaction -E snowflake -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: | poetry run pytest tests/load if: runner.os != 'Windows' diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml index 3e694d847a..83800fa789 100644 --- a/.github/workflows/test_destination_synapse.yml +++ b/.github/workflows/test_destination_synapse.yml @@ -9,6 +9,7 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} RUNTIME__SENTRY_DSN: https://cf6086f7d263462088b9fb9f9947caee@o4505514867163136.ingest.sentry.io/4505516212682752 RUNTIME__LOG_LEVEL: ERROR @@ -70,6 +71,9 @@ jobs: - name: Install dependencies run: poetry install --no-interaction -E synapse -E s3 -E gs -E az --with sentry-sdk --with pipeline + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: | poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py if: runner.os != 'Windows' diff --git a/.github/workflows/test_destination_weaviate.yml b/.github/workflows/test_destination_weaviate.yml index 1f242f8b2e..8c0161f364 100644 --- a/.github/workflows/test_destination_weaviate.yml +++ b/.github/workflows/test_destination_weaviate.yml @@ -8,6 +8,8 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR @@ -56,6 +58,9 @@ jobs: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - name: Install dependencies run: poetry install --no-interaction -E weaviate -E parquet --with sentry-sdk --with pipeline - run: | diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index 8787a8986b..c60d870b05 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -10,6 +10,8 @@ on: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + # For s3 compatible tests TESTS__R2_AWS_ACCESS_KEY_ID: a4950a5003b26f5a71ac97ef3848ff4c TESTS__R2_AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} @@ -71,8 +73,8 @@ jobs: # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction -E redshift -E gs -E s3 -E az -E parquet -E duckdb -E cli --with sentry-sdk --with pipeline - # - name: Install self - # run: poetry install --no-interaction + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | poetry run pytest tests/load diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index 4ddebbda3b..12f5c0ca48 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -11,6 +11,8 @@ on: workflow_dispatch: env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB @@ -84,6 +86,9 @@ jobs: - name: Install dependencies run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate --with sentry-sdk --with pipeline + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: poetry run pytest tests/load && poetry run pytest tests/cli name: Run tests Linux env: