Merge branch 'devel' into rfix/allows-naming-conventions

.github/workflows/deploy_docs.yml

@@ -1,4 +1,4 @@
-name: deploy docs
+name: docs | deploy docs
 
 on:
   schedule:
@@ -11,6 +11,7 @@ env:
 jobs:
   deploy:
     runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.head.repo.fork }}
     steps:
       - name: Trigger deploy hook
         run: curl ${{ env.NETLIFY_DOCS_PRODUCTION_DEPLOY_HOOK }} -X POST

.github/workflows/get_docs_changes.yml

@@ -1,4 +1,4 @@
-name: get docs changes
+name: util | get docs changes
 
 on:
   workflow_call:
@@ -13,6 +13,7 @@ env:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     runs-on: ubuntu-latest
     outputs:
       changes_outside_docs: ${{ steps.check_changes.outputs.changes_outside_docs }}

.github/workflows/lint.yml

@@ -1,5 +1,5 @@
 
-name: lint
+name: lint | code & tests
 
 on:
   pull_request:
@@ -8,18 +8,24 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
   run_lint:
-    name: Lint
+    name: lint
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        os:
+          - ubuntu-latest
         python-version: ["3.8.x", "3.9.x", "3.10.x", "3.11.x"]
 
     defaults:
@@ -60,19 +66,13 @@ jobs:
           export PATH=$PATH:"/c/Program Files/usr/bin" # needed for Windows
           make lint
 
-      # - name: print envs
-      #   run: |
-      #     echo "The GitHub Actor's username is: $GITHUB_ACTOR"
-      #     echo "The GitHub repo owner is: $GITHUB_REPOSITORY_OWNER"
-      #     echo "The GitHub repo is: $GITHUB_REPOSITORY"
-
   matrix_job_required_check:
-    name: Lint results
+    name: lint | code & tests
     needs: run_lint
     runs-on: ubuntu-latest
     if: always()
     steps:
       - name: Check matrix job results
         if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
         run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1      
+          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1

.github/workflows/test_airflow.yml

@@ -1,4 +1,4 @@
-name: test airflow integration
+name: tools | airflow
 
 on:
   pull_request:
@@ -7,12 +7,17 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
   run_airflow:
-    name: Tests Airflow integration
+    name: tools | airflow tests
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     runs-on: ubuntu-latest

.github/workflows/test_build_images.yml

@@ -1,4 +1,4 @@
-name: test build docker images
+name: tools | docker images
 
 on:
   pull_request:
@@ -7,12 +7,17 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
   run_airflow:
-    name: Build alpine and airflow images
+    name: tools | docker images build
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     runs-on: ubuntu-latest
@@ -34,4 +39,4 @@ jobs:
           installer-parallel: true
 
       - name: Build images
-        run: make test-build-images
+        run: make test-build-images

.github/workflows/test_common.yml

@@ -1,5 +1,4 @@
-
-name: test common
+name: common | common
 
 on:
   pull_request:
@@ -8,15 +7,21 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   RUNTIME__LOG_LEVEL: ERROR
+  RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
   run_common:
-    name: Tests common dlt code
+    name: test
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
@@ -49,6 +54,18 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Install tzdata on windows
+        run: |
+          cd %USERPROFILE%
+          curl https://data.iana.org/time-zones/releases/tzdata2021e.tar.gz --output tzdata.tar.gz
+          mkdir tzdata
+          tar --extract --file tzdata.tar.gz --directory tzdata
+          mkdir %USERPROFILE%\Downloads\tzdata
+          copy tzdata %USERPROFILE%\Downloads\tzdata
+          curl https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/windowsZones.xml --output %USERPROFILE%\Downloads\tzdata\windowsZones.xml
+        if: runner.os == 'Windows'
+        shell: cmd
+
       - name: Install Poetry
         # https://github.com/snok/install-poetry#running-on-windows
         uses: snok/[email protected]
@@ -87,11 +104,24 @@ jobs:
         if: runner.os != 'Windows'
         name: Run pipeline smoke tests with minimum deps Linux/MAC
       - run: |
-          poetry run pytest tests/pipeline/test_pipeline.py tests/pipeline/test_import_export_schema.py
+          poetry run pytest tests/pipeline/test_pipeline.py tests/pipeline/test_import_export_schema.py -m "not forked"
         if: runner.os == 'Windows'
         name: Run smoke tests with minimum deps Windows
         shell: cmd
 
+      - name: Install pyarrow
+        run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk
+
+      - run: |
+          poetry run pytest tests/pipeline/test_pipeline_extra.py -k arrow
+        if: runner.os != 'Windows'
+        name: Run pipeline tests with pyarrow but no pandas installed
+      - run: |
+          poetry run pytest tests/pipeline/test_pipeline_extra.py -k arrow  -m "not forked"
+        if: runner.os == 'Windows'
+        name: Run pipeline tests with pyarrow but no pandas installed Windows
+        shell: cmd
+
       - name: Install pipeline dependencies
         run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk --with pipeline
 
@@ -100,7 +130,7 @@ jobs:
         if: runner.os != 'Windows'
         name: Run extract and pipeline tests Linux/MAC
       - run: |
-          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations
+          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations  -m "not forked"
         if: runner.os == 'Windows'
         name: Run extract tests Windows
         shell: cmd
@@ -119,7 +149,7 @@ jobs:
       #   shell: cmd
 
   matrix_job_required_check:
-    name: Common tests
+    name: common | common tests
     needs: run_common
     runs-on: ubuntu-latest
     if: always()

.github/workflows/test_dbt_cloud.yml

@@ -1,5 +1,5 @@
 
-name: test dbt cloud
+name: tools | dbt cloud
 
 on:
   pull_request:
@@ -8,32 +8,33 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   # all credentials must be present to be passed to dbt cloud
   DBT_CLOUD__ACCOUNT_ID: ${{ secrets.DBT_CLOUD__ACCOUNT_ID }}
   DBT_CLOUD__JOB_ID: ${{ secrets.DBT_CLOUD__JOB_ID }}
   DBT_CLOUD__API_TOKEN: ${{ secrets.DBT_CLOUD__API_TOKEN }}
 
   RUNTIME__LOG_LEVEL: ERROR
+  RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
-    if: ${{ !github.event.pull_request.head.repo.fork }}
+    if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}
 
   run_dbt_cloud:
-    name: Tests dbt cloud
+    name: tools | dbt cloud tests
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -66,21 +67,4 @@ jobs:
 
       - run: |
           poetry run pytest tests/helpers/dbt_cloud_tests -k '(not venv)'
-        if: runner.os != 'Windows'
         name: Run dbt cloud - Linux/MAC
-      - run: |
-          poetry run pytest tests/helpers/dbt_cloud_tests -k "(not venv)"
-        if: runner.os == 'Windows'
-        name: Run dbt cloud - Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: dbt cloud tests
-    needs: run_dbt_cloud
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1

.github/workflows/test_dbt_runner.yml

@@ -1,5 +1,5 @@
 
-name: test dbt runner
+name: tools | dbt runner
 
 on:
   pull_request:
@@ -8,29 +8,30 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
 
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
   RUNTIME__LOG_LEVEL: ERROR
+  RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
-    if: ${{ !github.event.pull_request.head.repo.fork }}
+    if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}
 
   run_dbt:
-    name: Tests dbt runner
+    name: tools | dbt runner tests
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -80,21 +81,4 @@ jobs:
 
       - run: |
           poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -k '(not local)'
-        if: runner.os != 'Windows'
         name: Run dbt runner with venv - Linux/MAC
-      - run: |
-          poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -m "not forked" -k "(not local)"
-        if: runner.os == 'Windows'
-        name: Run dbt runner with venv - Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: dbt runner tests
-    needs: run_dbt
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1