Merge branch 'devel' into d#/source_and_schema_changes

# Conflicts: # dlt/common/schema/schema.py # dlt/common/schema/utils.py # dlt/pipeline/pipeline.py # tests/common/cases/schemas/eth/ethereum_schema_v7.yml # tests/common/schema/test_schema.py # tests/common/schema/test_versioning.py # tests/common/utils.py # tests/extract/test_incremental.py
dlt-hub · Nov 21, 2023 · c8e6e25 · c8e6e25
2 parents 7cb566e + 3fb2c30
commit c8e6e25
Show file tree

Hide file tree

Showing 243 changed files with 6,165 additions and 3,228 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -45,7 +45,7 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --all-extras --with airflow
+        run: poetry install --no-interaction --all-extras --with airflow --with docs --with providers --with sentry-sdk --with pipeline
 
       # - name: Install self
       #   run: poetry install --no-interaction

diff --git a/.github/workflows/test_airflow.yml b/.github/workflows/test_airflow.yml
@@ -41,7 +41,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-airflow-runner
 
       - name: Install dependencies
-        run: poetry install --no-interaction --with airflow -E duckdb -E parquet
+        run: poetry install --no-interaction --with airflow --with pipeline -E duckdb -E parquet --with sentry-sdk
 
       - run: |
           poetry run pytest tests/helpers/airflow_tests

diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml
@@ -55,40 +55,67 @@ jobs:
           virtualenvs-in-project: true
           installer-parallel: true
 
-      - name: Load cached venv
-        id: cached-poetry-dependencies
-        uses: actions/cache@v3
-        with:
-          # path: ${{ steps.pip-cache.outputs.dir }}
-          path: .venv
-          key: venv-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+      # NOTE: do not cache. we want to have a clean state each run and we upgrade depdendencies later
+      # - name: Load cached venv
+      #   id: cached-poetry-dependencies
+      #   uses: actions/cache@v3
+      #   with:
+      #     # path: ${{ steps.pip-cache.outputs.dir }}
+      #     path: .venv
+      #     key: venv-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        run: poetry install --no-interaction --with sentry-sdk
+
+      - run: |
+          poetry run pytest tests/common tests/normalize tests/reflection tests/sources tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py
+        if: runner.os != 'Windows'
+        name: Run common tests with minimum dependencies Linux/MAC
+      - run: |
+          poetry run pytest tests/common tests/normalize tests/reflection tests/sources tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py -m "not forked"
+        if: runner.os == 'Windows'
+        name: Run common tests with minimum dependencies Windows
+        shell: cmd
 
-      - name: Install dependencies + sentry
-        run: poetry install --no-interaction  -E parquet -E pydantic && pip install sentry-sdk
+      - name: Install duckdb dependencies
+        run: poetry install --no-interaction -E duckdb --with sentry-sdk
 
       - run: |
-          poetry run pytest tests/common tests/normalize tests/reflection tests/sources
+          poetry run pytest tests/pipeline/test_pipeline.py
         if: runner.os != 'Windows'
-        name: Run tests Linux/MAC
+        name: Run pipeline smoke tests with minimum deps Linux/MAC
       - run: |
-          poetry run pytest tests/common tests/normalize tests/reflection tests/sources -m "not forked"
+          poetry run pytest tests/pipeline/test_pipeline.py
         if: runner.os == 'Windows'
-        name: Run tests Windows
+        name: Run smoke tests with minimum deps Windows
         shell: cmd
 
-      - name: Install extra dependencies
-        run: poetry install --no-interaction -E duckdb -E cli -E parquet -E pydantic
+      - name: Install pipeline dependencies
+        run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk --with pipeline
 
       - run: |
-          poetry run pytest tests/extract tests/pipeline tests/cli/common
+          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common
         if: runner.os != 'Windows'
-        name: Run extra tests Linux/MAC
+        name: Run extract and pipeline tests Linux/MAC
       - run: |
-          poetry run pytest tests/extract tests/pipeline tests/cli/common
+          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common
         if: runner.os == 'Windows'
-        name: Run extra tests Windows
+        name: Run extract tests Windows
         shell: cmd
 
+      # - name: Install Pydantic 1.0
+      #   run: pip install "pydantic<2"
+
+      # - run: |
+      #     poetry run pytest tests/libs
+      #   if: runner.os != 'Windows'
+      #   name: Run extract and pipeline tests Linux/MAC
+      # - run: |
+      #     poetry run pytest tests/libs
+      #   if: runner.os == 'Windows'
+      #   name: Run extract tests Windows
+      #   shell: cmd
+
   matrix_job_required_check:
     name: Common tests
     needs: run_common

diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml
@@ -68,7 +68,7 @@ jobs:
 
       - name: Install dependencies
         # install dlt with postgres support
-        run: poetry install --no-interaction -E postgres -E dbt
+        run: poetry install --no-interaction -E postgres -E dbt --with sentry-sdk
 
       - run: |
           poetry run pytest tests/helpers/dbt_tests -k '(not venv)'

diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml
@@ -70,7 +70,7 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E athena
+        run: poetry install --no-interaction -E athena --with sentry-sdk --with pipeline
 
       - run: |
           poetry run pytest tests/load

diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml
@@ -70,7 +70,7 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E athena
+        run: poetry install --no-interaction -E  --with sentry-sdk --with pipeline
 
       - run: |
           poetry run pytest tests/load

diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml
@@ -79,7 +79,7 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E bigquery --with providers -E parquet
+        run: poetry install --no-interaction -E bigquery --with providers -E parquet --with sentry-sdk --with pipeline
 
       # - name: Install self
       #   run: poetry install --no-interaction

diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml
@@ -65,7 +65,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E mssql -E s3 -E gs -E az -E parquet
+        run: poetry install --no-interaction -E mssql -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline
 
       - run: |
           poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py

diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml
@@ -59,7 +59,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E qdrant -E parquet
+        run: poetry install --no-interaction -E qdrant -E parquet --with sentry-sdk --with pipeline
       - run: |
           poetry run pytest tests/load/
         if: runner.os != 'Windows'

diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml
@@ -71,7 +71,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E snowflake -E s3 -E gs -E az
+        run: poetry install --no-interaction -E snowflake -E s3 -E gs -E az --with sentry-sdk --with pipeline
 
       - run: |
           poetry run pytest tests/load

diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml
@@ -5,9 +5,9 @@ on:
     branches:
       - master
       - devel
-      
+
   workflow_dispatch:
-  
+
 env:
   DESTINATION__SYNAPSE__CREDENTIALS: ${{ secrets.SYNAPSE_CREDENTIALS }}
   DESTINATION__SYNAPSE__CREDENTIALS__PASSWORD: ${{ secrets.SYNAPSE_PASSWORD }}
@@ -42,7 +42,7 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      
+
       - name: Check out
         uses: actions/checkout@master
 
@@ -70,7 +70,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E synapse -E s3 -E gs -E az
+        run: poetry install --no-interaction -E synapse -E s3 -E gs -E az --with sentry-sdk --with pipeline
 
       - run: |
           poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py

diff --git a/.github/workflows/test_destination_weaviate.yml b/.github/workflows/test_destination_weaviate.yml
@@ -61,7 +61,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E weaviate -E parquet
+        run: poetry install --no-interaction -E weaviate -E parquet --with sentry-sdk --with pipeline
       - run: |
           poetry run pytest tests/load/
         if: runner.os != 'Windows'

diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml
@@ -87,7 +87,7 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E redshift -E gs -E s3 -E az -E parquet -E duckdb -E cli
+        run: poetry install --no-interaction -E redshift -E gs -E s3 -E az -E parquet -E duckdb -E cli --with sentry-sdk --with pipeline
 
       # - name: Install self
       #   run: poetry install --no-interaction

diff --git a/.github/workflows/test_doc_snippets.yml b/.github/workflows/test_doc_snippets.yml
@@ -59,7 +59,7 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E duckdb -E weaviate -E parquet --with docs --without airflow
+        run: poetry install --no-interaction -E duckdb -E weaviate -E parquet --with docs --without airflow --with sentry-sdk --with pipeline
 
       - name: Run linter and tests
         run: make test-and-lint-snippets

diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml
@@ -84,7 +84,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-local-destinations
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate
+        run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate --with sentry-sdk --with pipeline
 
       - run: poetry run pytest tests/load && poetry run pytest tests/cli
         name: Run tests Linux

diff --git a/Makefile b/Makefile
@@ -44,7 +44,7 @@ has-poetry:
 	poetry --version
 
 dev: has-poetry
-	poetry install --all-extras --with airflow --with docs --with providers
+	poetry install --all-extras --with airflow --with docs --with providers --with pipeline --with sentry-sdk
 
 lint:
 	./check-package.sh
@@ -87,3 +87,4 @@ test-build-images: build-library
 	grep `cat compiled_packages.txt` _gen_requirements.txt > compiled_requirements.txt
 	docker build -f deploy/dlt/Dockerfile.airflow --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" .
 	docker build -f deploy/dlt/Dockerfile --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" .
+
diff --git a/dlt/__init__.py b/dlt/__init__.py
@@ -31,6 +31,7 @@
 from dlt.extract.decorators import source, resource, transformer, defer
 from dlt.pipeline import pipeline as _pipeline, run, attach, Pipeline, dbt, current as _current, mark as _mark
 from dlt.pipeline import progress
+from dlt import destinations
 
 pipeline = _pipeline
 current = _current
@@ -64,4 +65,5 @@
     "TSecretValue",
     "TCredentials",
     "sources",
+    "destinations",
 ]