Merge pull request #1200 from dlt-hub/devel

master merge for 0.4.8
dlt-hub · Apr 9, 2024 · c99d612 · c99d612
2 parents e9c8f61 + 7939749
commit c99d612
Show file tree

Hide file tree

Showing 560 changed files with 11,014 additions and 6,843 deletions.
diff --git a/.github/workflows/deploy_docs.yml b/.github/workflows/deploy_docs.yml
@@ -1,4 +1,4 @@
-name: deploy docs
+name: docs | deploy docs
 
 on:
   schedule:
@@ -11,6 +11,7 @@ env:
 jobs:
   deploy:
     runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.head.repo.fork }}
     steps:
       - name: Trigger deploy hook
         run: curl ${{ env.NETLIFY_DOCS_PRODUCTION_DEPLOY_HOOK }} -X POST
diff --git a/.github/workflows/get_docs_changes.yml b/.github/workflows/get_docs_changes.yml
@@ -1,4 +1,4 @@
-name: get docs changes
+name: util | get docs changes
 
 on:
   workflow_call:
@@ -13,6 +13,7 @@ env:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     runs-on: ubuntu-latest
     outputs:
       changes_outside_docs: ${{ steps.check_changes.outputs.changes_outside_docs }}

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -1,5 +1,5 @@
 
-name: lint
+name: lint | code & tests
 
 on:
   pull_request:
@@ -14,10 +14,11 @@ concurrency:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
   run_lint:
-    name: Lint
+    name: lint
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
@@ -65,14 +66,8 @@ jobs:
           export PATH=$PATH:"/c/Program Files/usr/bin" # needed for Windows
           make lint
 
-      # - name: print envs
-      #   run: |
-      #     echo "The GitHub Actor's username is: $GITHUB_ACTOR"
-      #     echo "The GitHub repo owner is: $GITHUB_REPOSITORY_OWNER"
-      #     echo "The GitHub repo is: $GITHUB_REPOSITORY"
-
   matrix_job_required_check:
-    name: Lint results
+    name: lint | code & tests
     needs: run_lint
     runs-on: ubuntu-latest
     if: always()

diff --git a/.github/workflows/test_airflow.yml b/.github/workflows/test_airflow.yml
@@ -1,4 +1,4 @@
-name: test airflow integration
+name: tools | airflow
 
 on:
   pull_request:
@@ -13,10 +13,11 @@ concurrency:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
   run_airflow:
-    name: Tests Airflow integration
+    name: tools | airflow tests
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     runs-on: ubuntu-latest

diff --git a/.github/workflows/test_build_images.yml b/.github/workflows/test_build_images.yml
@@ -1,4 +1,4 @@
-name: test build docker images
+name: tools | docker images
 
 on:
   pull_request:
@@ -13,10 +13,11 @@ concurrency:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
   run_airflow:
-    name: Build alpine and airflow images
+    name: tools | docker images build
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     runs-on: ubuntu-latest
@@ -38,4 +39,4 @@ jobs:
           installer-parallel: true
 
       - name: Build images
-        run: make test-build-images
+        run: make test-build-images
diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml
@@ -1,4 +1,4 @@
-name: test common
+name: common | common
 
 on:
   pull_request:
@@ -16,10 +16,11 @@ env:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
   run_common:
-    name: Tests common dlt code
+    name: test
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
@@ -52,6 +53,18 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Install tzdata on windows
+        run: |
+          cd %USERPROFILE%
+          curl https://data.iana.org/time-zones/releases/tzdata2021e.tar.gz --output tzdata.tar.gz
+          mkdir tzdata
+          tar --extract --file tzdata.tar.gz --directory tzdata
+          mkdir %USERPROFILE%\Downloads\tzdata
+          copy tzdata %USERPROFILE%\Downloads\tzdata
+          curl https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/windowsZones.xml --output %USERPROFILE%\Downloads\tzdata\windowsZones.xml
+        if: runner.os == 'Windows'
+        shell: cmd
+
       - name: Install Poetry
         # https://github.com/snok/install-poetry#running-on-windows
         uses: snok/[email protected]
@@ -135,7 +148,7 @@ jobs:
       #   shell: cmd
 
   matrix_job_required_check:
-    name: Common tests
+    name: common | common tests
     needs: run_common
     runs-on: ubuntu-latest
     if: always()

diff --git a/.github/workflows/test_dbt_cloud.yml b/.github/workflows/test_dbt_cloud.yml
@@ -1,5 +1,5 @@
 
-name: test dbt cloud
+name: tools | dbt cloud
 
 on:
   pull_request:
@@ -22,22 +22,18 @@ env:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
-    if: ${{ !github.event.pull_request.head.repo.fork }}
+    if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}
 
   run_dbt_cloud:
-    name: Tests dbt cloud
+    name: tools | dbt cloud tests
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -70,21 +66,4 @@ jobs:
 
       - run: |
           poetry run pytest tests/helpers/dbt_cloud_tests -k '(not venv)'
-        if: runner.os != 'Windows'
         name: Run dbt cloud - Linux/MAC
-      - run: |
-          poetry run pytest tests/helpers/dbt_cloud_tests -k "(not venv)"
-        if: runner.os == 'Windows'
-        name: Run dbt cloud - Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: dbt cloud tests
-    needs: run_dbt_cloud
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml
@@ -1,5 +1,5 @@
 
-name: test dbt runner
+name: tools | dbt runner
 
 on:
   pull_request:
@@ -19,22 +19,18 @@ env:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
-    if: ${{ !github.event.pull_request.head.repo.fork }}
+    if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}
 
   run_dbt:
-    name: Tests dbt runner
+    name: tools | dbt runner tests
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -84,21 +80,4 @@ jobs:
 
       - run: |
           poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -k '(not local)'
-        if: runner.os != 'Windows'
         name: Run dbt runner with venv - Linux/MAC
-      - run: |
-          poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -m "not forked" -k "(not local)"
-        if: runner.os == 'Windows'
-        name: Run dbt runner with venv - Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: dbt runner tests
-    needs: run_dbt
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml
@@ -1,12 +1,14 @@
 
-name: test athena
+name: dest | athena
 
 on:
   pull_request:
     branches:
       - master
       - devel
   workflow_dispatch:
+  schedule:
+    - cron:  '0 2 * * *'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -20,27 +22,23 @@ env:
   RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB
   ACTIVE_DESTINATIONS: "[\"athena\"]"
   ALL_FILESYSTEM_DRIVERS: "[\"memory\"]"
-  EXCLUDED_DESTINATION_CONFIGURATIONS: "[\"athena-parquet-staging-iceberg\"]"
+  EXCLUDED_DESTINATION_CONFIGURATIONS: "[\"athena-parquet-staging-iceberg\", \"athena-parquet-no-staging-iceberg\"]"
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
     # Tests that require credentials do not run in forks
-    if: ${{ !github.event.pull_request.head.repo.fork }}
+    if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}
 
   run_loader:
-    name: test destination athena
+    name: dest | athena tests
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -75,22 +73,11 @@ jobs:
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       - run: |
-          poetry run pytest tests/load
-        if: runner.os != 'Windows'
-        name: Run tests Linux/MAC
+          poetry run pytest tests/load -m "essential"
+        name: Run essential tests Linux
+        if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || !github.event_name == 'schedule')}}
+
       - run: |
           poetry run pytest tests/load
-        if: runner.os == 'Windows'
-        name: Run tests Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: Redshift, PostgreSQL and DuckDB tests
-    needs: run_loader
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
+        name: Run all tests Linux
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}}