Skip to content

Commit b98e542

Browse files
adrianbradrianbr
authored
Merge branch 'master' into docs/run_monitoring_update
2 parents dc95e6b + 40e6a89 commit b98e542

File tree

224 files changed

+8915
-1957
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

224 files changed

+8915
-1957
lines changed

‎.github/ISSUE_TEMPLATE/documentation_request.yml

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
---
2+
name: Documentation Request
3+
description: Suggest improvements or additions to dlt documentation
4+
labels: ["kind:documentation"]
5+
body:
6+
- type: markdown
7+
attributes:
8+
value: |
9+
Thanks for contributing to the dlt documentation!
10+
For significant updates or new content, open an issue to discuss your ideas. This helps us
11+
coordinate and integrate your suggestions effectively. For simple edits like typos, you can
12+
directly submit a [pull request](https://github.com/dlt-hub/dlt/pulls).
13+
- type: textarea
14+
attributes:
15+
label: Documentation description
16+
description: Describe the documentation improvements or additions you'd like to see.
17+
placeholder: >
18+
What specific documentation changes are you suggesting?
19+
- type: dropdown
20+
attributes:
21+
label: Are you a dlt user?
22+
description: Please tell us how you use dlt and how this impacts your need for documentation.
23+
multiple: false
24+
options:
25+
- "Yes, I'm already a dlt user."
26+
- "Yes, I run dlt in production."
27+
- "Yes, I use it for fun."
28+
- "I'm considering using dlt, but the documentation is unclear."

‎.github/workflows/deploy_docs.yml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
name: deploy docs
2+
3+
on:
4+
schedule:
5+
- cron: '0 2 * * *'
6+
workflow_dispatch:
7+
8+
env:
9+
NETLIFY_DOCS_PRODUCTION_DEPLOY_HOOK: ${{ secrets.NETLIFY_DOCS_PRODUCTION_DEPLOY_HOOK }}
10+
11+
jobs:
12+
deploy:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Trigger deploy hook
16+
run: curl ${{ env.NETLIFY_DOCS_PRODUCTION_DEPLOY_HOOK }} -X POST

‎.github/workflows/lint.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ jobs:
2525
defaults:
2626
run:
2727
shell: bash
28-
runs-on: ${{ matrix.os }}
28+
runs-on: ${{ matrix.os }}
2929

3030
steps:
3131

@@ -42,7 +42,7 @@ jobs:
4242
with:
4343
virtualenvs-create: true
4444
virtualenvs-in-project: true
45-
installer-parallel: true
45+
installer-parallel: true
4646

4747
- name: Load cached venv
4848
id: cached-poetry-dependencies
@@ -57,7 +57,7 @@ jobs:
5757

5858
- name: Run make lint
5959
run: |
60-
export PATH=$PATH:"/c/Program Files/usr/bin" # needed for Windows
60+
export PATH=$PATH:"/c/Program Files/usr/bin" # needed for Windows
6161
make lint
6262
6363
# - name: print envs

‎.github/workflows/test_dbt_runner.yml

+4-10
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,8 @@ on:
99
workflow_dispatch:
1010

1111
env:
12-
# all credentials must be present to be passed to dbt runner
13-
DESTINATION__POSTGRES__CREDENTIALS: postgresql://[email protected]:5432/dlt_data
14-
DESTINATION__REDSHIFT__CREDENTIALS: postgresql://[email protected]:5439/dlt_ci
15-
DESTINATION__SNOWFLAKE__CREDENTIALS: snowflake://loader@kgiotue-wn98412/dlt_data?warehouse=COMPUTE_WH&role=DLT_LOADER_ROLE
16-
DESTINATION__CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }}
17-
18-
DESTINATION__CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci
19-
DESTINATION__CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com
20-
DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }}
21-
DESTINATION__CREDENTIALS__TOKEN_URI: https://oauth2.googleapis.com/token
2212

13+
DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
2314
RUNTIME__LOG_LEVEL: ERROR
2415

2516
jobs:
@@ -70,6 +61,9 @@ jobs:
7061
# install dlt with postgres support
7162
run: poetry install --no-interaction -E postgres -E dbt --with sentry-sdk
7263

64+
- name: create secrets.toml
65+
run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
66+
7367
- run: |
7468
poetry run pytest tests/helpers/dbt_tests -k '(not venv)'
7569
if: runner.os != 'Windows'

‎.github/workflows/test_destination_athena.yml

+4-6
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,7 @@ on:
99
workflow_dispatch:
1010

1111
env:
12-
DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
13-
DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
14-
DESTINATION__ATHENA__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
15-
DESTINATION__ATHENA__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
16-
DESTINATION__ATHENA__CREDENTIALS__REGION_NAME: eu-central-1
17-
DESTINATION__ATHENA__QUERY_RESULT_BUCKET: s3://dlt-athena-output
12+
DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
1813

1914
RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
2015
RUNTIME__LOG_LEVEL: ERROR
@@ -72,6 +67,9 @@ jobs:
7267
# if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
7368
run: poetry install --no-interaction -E athena --with sentry-sdk --with pipeline
7469

70+
- name: create secrets.toml
71+
run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
72+
7573
- run: |
7674
poetry run pytest tests/load
7775
if: runner.os != 'Windows'

‎.github/workflows/test_destination_athena_iceberg.yml

+5-7
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,7 @@ on:
99
workflow_dispatch:
1010

1111
env:
12-
DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
13-
DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
14-
DESTINATION__ATHENA__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
15-
DESTINATION__ATHENA__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
16-
DESTINATION__ATHENA__CREDENTIALS__REGION_NAME: eu-central-1
17-
DESTINATION__ATHENA__QUERY_RESULT_BUCKET: s3://dlt-athena-output
12+
DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
1813

1914
RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
2015
RUNTIME__LOG_LEVEL: ERROR
@@ -70,7 +65,10 @@ jobs:
7065

7166
- name: Install dependencies
7267
# if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
73-
run: poetry install --no-interaction -E --with sentry-sdk --with pipeline
68+
run: poetry install --no-interaction -E --with sentry-sdk --with pipeline
69+
70+
- name: create secrets.toml
71+
run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
7472

7573
- run: |
7674
poetry run pytest tests/load

‎.github/workflows/test_destination_bigquery.yml

+3-13
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,7 @@ on:
99
workflow_dispatch:
1010

1111
env:
12-
CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci
13-
CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com
14-
CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }}
15-
CREDENTIALS__TOKEN_URI: https://oauth2.googleapis.com/token
16-
CREDENTIALS__CLIENT_ID: 929384042504-3mtjaj1s7vuvf53j88mgdq4te7akkjm3.apps.googleusercontent.com
17-
CREDENTIALS__CLIENT_SECRET: ${{ secrets.CREDENTIALS__CLIENT_SECRET }}
18-
CREDENTIALS__REFRESH_TOKEN: ${{ secrets.CREDENTIALS__REFRESH_TOKEN }}
19-
20-
# needed for bigquery staging tests
21-
# DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
22-
# DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
12+
DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
2313

2414
RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
2515
RUNTIME__LOG_LEVEL: ERROR
@@ -81,8 +71,8 @@ jobs:
8171
# if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
8272
run: poetry install --no-interaction -E bigquery --with providers -E parquet --with sentry-sdk --with pipeline
8373

84-
# - name: Install self
85-
# run: poetry install --no-interaction
74+
- name: create secrets.toml
75+
run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
8676

8777
- run: |
8878
poetry run pytest tests/helpers/providers tests/load

‎.github/workflows/test_destination_weaviate.yml ‎.github/workflows/test_destination_databricks.yml

+17-13
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
name: test weaviate
1+
2+
name: test databricks
23

34
on:
45
pull_request:
@@ -8,14 +9,12 @@ on:
89
workflow_dispatch:
910

1011
env:
11-
DESTINATION__WEAVIATE__CREDENTIALS__URL: ${{ secrets.DESTINATION__WEAVIATE__CREDENTIALS__URL }}
12-
DESTINATION__WEAVIATE__CREDENTIALS__API_KEY: ${{ secrets.DESTINATION__WEAVIATE__CREDENTIALS__API_KEY }}
13-
DESTINATION__WEAVIATE__CREDENTIALS__ADDITIONAL_HEADERS: ${{ secrets.DESTINATION__WEAVIATE__CREDENTIALS__ADDITIONAL_HEADERS }}
12+
DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
1413

1514
RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
1615
RUNTIME__LOG_LEVEL: ERROR
1716

18-
ACTIVE_DESTINATIONS: "[\"weaviate\"]"
17+
ACTIVE_DESTINATIONS: "[\"databricks\"]"
1918
ALL_FILESYSTEM_DRIVERS: "[\"memory\"]"
2019

2120
jobs:
@@ -24,20 +23,21 @@ jobs:
2423
if: ${{ !github.event.pull_request.head.repo.fork }}
2524

2625
run_loader:
27-
name: Tests Weaviate loader
26+
name: Tests Databricks loader
2827
needs: get_docs_changes
29-
if: !always()
30-
# if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
28+
if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
3129
strategy:
3230
fail-fast: false
3331
matrix:
34-
os: ["ubuntu-latest", "macos-latest", "windows-latest"]
32+
os: ["ubuntu-latest"]
33+
# os: ["ubuntu-latest", "macos-latest", "windows-latest"]
3534
defaults:
3635
run:
3736
shell: bash
3837
runs-on: ${{ matrix.os }}
3938

4039
steps:
40+
4141
- name: Check out
4242
uses: actions/checkout@master
4343

@@ -61,19 +61,23 @@ jobs:
6161
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
6262

6363
- name: Install dependencies
64-
run: poetry install --no-interaction -E weaviate -E parquet --with sentry-sdk --with pipeline
64+
run: poetry install --no-interaction -E databricks -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline
65+
66+
- name: create secrets.toml
67+
run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
68+
6569
- run: |
66-
poetry run pytest tests/load/
70+
poetry run pytest tests/load
6771
if: runner.os != 'Windows'
6872
name: Run tests Linux/MAC
6973
- run: |
70-
poetry run pytest tests/load/
74+
poetry run pytest tests/load
7175
if: runner.os == 'Windows'
7276
name: Run tests Windows
7377
shell: cmd
7478
7579
matrix_job_required_check:
76-
name: Weaviate loader tests
80+
name: Databricks loader tests
7781
needs: run_loader
7882
runs-on: ubuntu-latest
7983
if: always()

‎.github/workflows/test_destination_mssql.yml

+7-4
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ on:
99
workflow_dispatch:
1010

1111
env:
12-
DESTINATION__MSSQL__CREDENTIALS: mssql://[email protected]:1433/dlt_ci
13-
DESTINATION__MSSQL__CREDENTIALS__PASSWORD: ${{ secrets.MSSQL_PASSWORD }}
12+
13+
DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
1414

1515
RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
1616
RUNTIME__LOG_LEVEL: ERROR
@@ -67,12 +67,15 @@ jobs:
6767
- name: Install dependencies
6868
run: poetry install --no-interaction -E mssql -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline
6969

70+
- name: create secrets.toml
71+
run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
72+
7073
- run: |
71-
poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py
74+
poetry run pytest tests/load
7275
if: runner.os != 'Windows'
7376
name: Run tests Linux/MAC
7477
- run: |
75-
poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py
78+
poetry run pytest tests/load
7679
if: runner.os == 'Windows'
7780
name: Run tests Windows
7881
shell: cmd

‎.github/workflows/test_destination_qdrant.yml

+4-2
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@ on:
88
workflow_dispatch:
99

1010
env:
11-
DESTINATION__QDRANT__CREDENTIALS__LOCATION: ${{ secrets.DESTINATION__QDRANT__CREDENTIALS__LOCATION }}
12-
DESTINATION__QDRANT__CREDENTIALS__API_KEY: ${{ secrets.DESTINATION__QDRANT__CREDENTIALS__API_KEY }}
11+
DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
1312

1413
RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
1514
RUNTIME__LOG_LEVEL: ERROR
@@ -58,6 +57,9 @@ jobs:
5857
path: .venv
5958
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
6059

60+
- name: create secrets.toml
61+
run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
62+
6163
- name: Install dependencies
6264
run: poetry install --no-interaction -E qdrant -E parquet --with sentry-sdk --with pipeline
6365
- run: |

‎.github/workflows/test_destination_snowflake.yml

+4-11
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,7 @@ on:
99
workflow_dispatch:
1010

1111
env:
12-
DESTINATION__SNOWFLAKE__CREDENTIALS: snowflake://loader@kgiotue-wn98412/dlt_data?warehouse=COMPUTE_WH&role=DLT_LOADER_ROLE
13-
CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }}
14-
15-
# needed for snowflake staging tests
16-
DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
17-
DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
18-
DESTINATION__FILESYSTEM__CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci
19-
DESTINATION__FILESYSTEM__CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com
20-
DESTINATION__FILESYSTEM__CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }}
21-
DESTINATION__FILESYSTEM__CREDENTIALS__AZURE_STORAGE_ACCOUNT_NAME: dltdata
22-
DESTINATION__FILESYSTEM__CREDENTIALS__AZURE_STORAGE_ACCOUNT_KEY: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }}
12+
DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
2313

2414
RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
2515
RUNTIME__LOG_LEVEL: ERROR
@@ -73,6 +63,9 @@ jobs:
7363
- name: Install dependencies
7464
run: poetry install --no-interaction -E snowflake -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline
7565

66+
- name: create secrets.toml
67+
run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
68+
7669
- run: |
7770
poetry run pytest tests/load
7871
if: runner.os != 'Windows'

‎.github/workflows/test_destination_synapse.yml

+12-16
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,10 @@ on:
55
branches:
66
- master
77
- devel
8-
98
workflow_dispatch:
109

1110
env:
12-
DESTINATION__SYNAPSE__CREDENTIALS: ${{ secrets.SYNAPSE_CREDENTIALS }}
13-
DESTINATION__SYNAPSE__CREDENTIALS__PASSWORD: ${{ secrets.SYNAPSE_PASSWORD }}
11+
DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
1412

1513
RUNTIME__SENTRY_DSN: https://cf6086f7d263462088b9fb9f9947caee@o4505514867163136.ingest.sentry.io/4505516212682752
1614
RUNTIME__LOG_LEVEL: ERROR
@@ -19,19 +17,14 @@ env:
1917
ALL_FILESYSTEM_DRIVERS: "[\"memory\"]"
2018

2119
jobs:
22-
23-
build:
24-
runs-on: ubuntu-latest
25-
26-
steps:
27-
- name: Check source branch name
28-
run: |
29-
if [[ "${{ github.head_ref }}" != "synapse" ]]; then
30-
exit 1
31-
fi
20+
get_docs_changes:
21+
uses: ./.github/workflows/get_docs_changes.yml
22+
if: ${{ !github.event.pull_request.head.repo.fork }}
3223

3324
run_loader:
3425
name: Tests Synapse loader
26+
needs: get_docs_changes
27+
if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
3528
strategy:
3629
fail-fast: false
3730
matrix:
@@ -70,14 +63,17 @@ jobs:
7063
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
7164

7265
- name: Install dependencies
73-
run: poetry install --no-interaction -E synapse -E s3 -E gs -E az --with sentry-sdk --with pipeline
66+
run: poetry install --no-interaction -E synapse -E parquet --with sentry-sdk --with pipeline
67+
68+
- name: create secrets.toml
69+
run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
7470

7571
- run: |
76-
poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py
72+
poetry run pytest tests/load
7773
if: runner.os != 'Windows'
7874
name: Run tests Linux/MAC
7975
- run: |
80-
poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py
76+
poetry run pytest tests/load
8177
if: runner.os == 'Windows'
8278
name: Run tests Windows
8379
shell: cmd

0 commit comments

Comments
 (0)