add missing secrets injection

.github/workflows/test_dbt_runner.yml

@@ -9,6 +9,9 @@ on:
   workflow_dispatch:
 
 env:
+
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+
   # all credentials must be present to be passed to dbt runner
   DESTINATION__POSTGRES__CREDENTIALS: postgresql://[email protected]:5432/dlt_data
   DESTINATION__REDSHIFT__CREDENTIALS: postgresql://[email protected]:5439/dlt_ci
@@ -84,6 +87,9 @@ jobs:
         # install dlt with postgres support
         run: poetry run pip uninstall dbt-core -y
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - run: |
           poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -k '(not local)'
         if: runner.os != 'Windows'

.github/workflows/test_destination_athena.yml

@@ -9,6 +9,8 @@ on:
   workflow_dispatch:
 
 env:
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB
@@ -65,6 +67,9 @@ jobs:
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: poetry install --no-interaction -E athena --with sentry-sdk --with pipeline
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - run: |
           poetry run pytest tests/load
         if: runner.os != 'Windows'

.github/workflows/test_destination_athena_iceberg.yml

@@ -9,6 +9,8 @@ on:
   workflow_dispatch:
 
 env:
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB
@@ -65,6 +67,9 @@ jobs:
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: poetry install --no-interaction -E  --with sentry-sdk --with pipeline
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - run: |
           poetry run pytest tests/load
         if: runner.os != 'Windows'

.github/workflows/test_destination_bigquery.yml

@@ -9,12 +9,13 @@ on:
   workflow_dispatch:
 
 env:
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+
   # TODO: migrate to google secrets
   CREDENTIALS__CLIENT_ID: 929384042504-3mtjaj1s7vuvf53j88mgdq4te7akkjm3.apps.googleusercontent.com
   CREDENTIALS__CLIENT_SECRET: ${{ secrets.CREDENTIALS__CLIENT_SECRET }}
   CREDENTIALS__REFRESH_TOKEN: ${{ secrets.CREDENTIALS__REFRESH_TOKEN }}
 
-
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
 
@@ -75,8 +76,8 @@ jobs:
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: poetry install --no-interaction -E bigquery --with providers -E parquet --with sentry-sdk --with pipeline
 
-      # - name: Install self
-      #   run: poetry install --no-interaction
+     - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       - run: |
           poetry run pytest tests/helpers/providers tests/load

.github/workflows/test_destination_mssql.yml

@@ -9,6 +9,9 @@ on:
   workflow_dispatch:
 
 env:
+
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
 
@@ -64,6 +67,9 @@ jobs:
       - name: Install dependencies
         run: poetry install --no-interaction -E mssql -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - run: |
           poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py
         if: runner.os != 'Windows'

.github/workflows/test_destination_qdrant.yml

@@ -8,6 +8,7 @@ on:
   workflow_dispatch:
 
 env:
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
@@ -56,6 +57,9 @@ jobs:
           path: .venv
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - name: Install dependencies
         run: poetry install --no-interaction -E qdrant -E parquet --with sentry-sdk --with pipeline
       - run: |

.github/workflows/test_destination_snowflake.yml

@@ -9,6 +9,7 @@ on:
   workflow_dispatch:
 
 env:
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
@@ -62,6 +63,9 @@ jobs:
       - name: Install dependencies
         run: poetry install --no-interaction -E snowflake -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - run: |
           poetry run pytest tests/load
         if: runner.os != 'Windows'

.github/workflows/test_destination_synapse.yml

@@ -9,6 +9,7 @@ on:
   workflow_dispatch:
 
 env:
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
   RUNTIME__SENTRY_DSN: https://cf6086f7d263462088b9fb9f9947caee@o4505514867163136.ingest.sentry.io/4505516212682752
   RUNTIME__LOG_LEVEL: ERROR
@@ -70,6 +71,9 @@ jobs:
       - name: Install dependencies
         run: poetry install --no-interaction -E synapse -E s3 -E gs -E az --with sentry-sdk --with pipeline
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - run: |
           poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py
         if: runner.os != 'Windows'

.github/workflows/test_destination_weaviate.yml

@@ -8,6 +8,8 @@ on:
   workflow_dispatch:
 
 env:
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
 
@@ -56,6 +58,9 @@ jobs:
           path: .venv
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - name: Install dependencies
         run: poetry install --no-interaction -E weaviate -E parquet --with sentry-sdk --with pipeline
       - run: |

.github/workflows/test_destinations.yml

@@ -10,6 +10,8 @@ on:
 
 env:
 
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+
   # For s3 compatible tests
   TESTS__R2_AWS_ACCESS_KEY_ID: a4950a5003b26f5a71ac97ef3848ff4c
   TESTS__R2_AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
@@ -71,8 +73,8 @@ jobs:
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: poetry install --no-interaction -E redshift -E gs -E s3 -E az -E parquet -E duckdb -E cli --with sentry-sdk --with pipeline
 
-      # - name: Install self
-      #   run: poetry install --no-interaction
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       - run: |
           poetry run pytest tests/load

.github/workflows/test_local_destinations.yml

@@ -11,6 +11,8 @@ on:
   workflow_dispatch:
 
 env:
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB
@@ -84,6 +86,9 @@ jobs:
       - name: Install dependencies
         run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate --with sentry-sdk --with pipeline
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
       - run: poetry run pytest tests/load && poetry run pytest tests/cli
         name: Run tests Linux
         env: