From 82ea4fdbb7df9fdfda22404387e70a0be38d79c4 Mon Sep 17 00:00:00 2001
From: Sultan Iman <354868+sultaniman@users.noreply.github.com>
Date: Tue, 19 Mar 2024 15:54:26 +0100
Subject: [PATCH] Cancel previous CI runs for PRs when new commits pushed
 (#1109)

* Cancel previous CI runs for PRs when new commits pushed

* Small readme example pipeline code formatting

* Include matrix.os in group

* Use max-parallel: 3 for lint and common tests

* Skip matrix.os for linting

* Skip matrix.os for destination workflows

* Revert common tests workflow

* Parallelize linting

* Use only ubuntu to run linting

* Run qdrant tests only on linux

* Remove redundant qdrant workflow checks for windows os
---
 .github/workflows/lint.yml                        | 10 ++++++++--
 .github/workflows/test_airflow.yml                |  4 ++++
 .github/workflows/test_build_images.yml           |  4 ++++
 .github/workflows/test_common.yml                 |  5 ++++-
 .github/workflows/test_dbt_cloud.yml              |  4 ++++
 .github/workflows/test_dbt_runner.yml             |  4 ++++
 .github/workflows/test_destination_athena.yml     |  4 ++++
 .../workflows/test_destination_athena_iceberg.yml |  4 ++++
 .github/workflows/test_destination_bigquery.yml   |  4 ++++
 .github/workflows/test_destination_databricks.yml |  4 ++++
 .github/workflows/test_destination_mssql.yml      |  4 ++++
 .github/workflows/test_destination_qdrant.yml     | 15 +++++++--------
 .github/workflows/test_destination_snowflake.yml  |  4 ++++
 .github/workflows/test_destination_synapse.yml    |  6 +++++-
 .github/workflows/test_destinations.yml           |  4 ++++
 .github/workflows/test_doc_snippets.yml           |  4 ++++
 .github/workflows/test_local_destinations.yml     |  4 ++++
 README.md                                         |  3 +++
 18 files changed, 79 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 35ccb71ab5..67b172db5e 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   get_docs_changes:
     uses: ./.github/workflows/get_docs_changes.yml
@@ -18,8 +22,10 @@ jobs:
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
       fail-fast: false
+      max-parallel: 14
       matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        os:
+          - ubuntu-latest
         python-version: ["3.8.x", "3.9.x", "3.10.x", "3.11.x"]
 
     defaults:
@@ -75,4 +81,4 @@ jobs:
       - name: Check matrix job results
         if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
         run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1      
+          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
diff --git a/.github/workflows/test_airflow.yml b/.github/workflows/test_airflow.yml
index bbed326344..02513618d6 100644
--- a/.github/workflows/test_airflow.yml
+++ b/.github/workflows/test_airflow.yml
@@ -7,6 +7,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   get_docs_changes:
     uses: ./.github/workflows/get_docs_changes.yml
diff --git a/.github/workflows/test_build_images.yml b/.github/workflows/test_build_images.yml
index 9668e23cb3..489d776f40 100644
--- a/.github/workflows/test_build_images.yml
+++ b/.github/workflows/test_build_images.yml
@@ -7,6 +7,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   get_docs_changes:
     uses: ./.github/workflows/get_docs_changes.yml
diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml
index 2160025ea0..40c77e962b 100644
--- a/.github/workflows/test_common.yml
+++ b/.github/workflows/test_common.yml
@@ -1,4 +1,3 @@
-
 name: test common
 
 on:
@@ -8,6 +7,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   RUNTIME__LOG_LEVEL: ERROR
 
diff --git a/.github/workflows/test_dbt_cloud.yml b/.github/workflows/test_dbt_cloud.yml
index 2d06ac96ba..a123e051e8 100644
--- a/.github/workflows/test_dbt_cloud.yml
+++ b/.github/workflows/test_dbt_cloud.yml
@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   # all credentials must be present to be passed to dbt cloud
   DBT_CLOUD__ACCOUNT_ID: ${{ secrets.DBT_CLOUD__ACCOUNT_ID }}
diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml
index 5ae791c979..1c425f14e9 100644
--- a/.github/workflows/test_dbt_runner.yml
+++ b/.github/workflows/test_dbt_runner.yml
@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
 
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml
index e9e17edefe..b94bdc6ee2 100644
--- a/.github/workflows/test_destination_athena.yml
+++ b/.github/workflows/test_destination_athena.yml
@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml
index d77e35f088..acb5f35dfd 100644
--- a/.github/workflows/test_destination_athena_iceberg.yml
+++ b/.github/workflows/test_destination_athena_iceberg.yml
@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml
index 00027768a5..d11f7155d4 100644
--- a/.github/workflows/test_destination_bigquery.yml
+++ b/.github/workflows/test_destination_bigquery.yml
@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml
index f301a1b9ed..2a2fa8e10d 100644
--- a/.github/workflows/test_destination_databricks.yml
+++ b/.github/workflows/test_destination_databricks.yml
@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml
index d1da25c067..f96c64219d 100644
--- a/.github/workflows/test_destination_mssql.yml
+++ b/.github/workflows/test_destination_mssql.yml
@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
 
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml
index 758c18b56b..3237801dbf 100644
--- a/.github/workflows/test_destination_qdrant.yml
+++ b/.github/workflows/test_destination_qdrant.yml
@@ -7,6 +7,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
@@ -28,7 +32,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        os:
+          - ubuntu-latest
     defaults:
       run:
         shell: bash
@@ -64,13 +69,7 @@ jobs:
         run: poetry install --no-interaction -E qdrant -E parquet --with sentry-sdk --with pipeline
       - run: |
           poetry run pytest tests/load/
-        if: runner.os != 'Windows'
-        name: Run tests Linux/MAC
-      - run: |
-          poetry run pytest tests/load/
-        if: runner.os == 'Windows'
-        name: Run tests Windows
-        shell: cmd
+        name: Run tests Linux
 
   matrix_job_required_check:
     name: Qdrant loader tests
diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml
index 979ea3e917..1ef290682c 100644
--- a/.github/workflows/test_destination_snowflake.yml
+++ b/.github/workflows/test_destination_snowflake.yml
@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml
index ecd890d32a..774c83314f 100644
--- a/.github/workflows/test_destination_synapse.yml
+++ b/.github/workflows/test_destination_synapse.yml
@@ -7,6 +7,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
@@ -24,7 +28,7 @@ jobs:
   run_loader:
     name: Tests Synapse loader
     needs: get_docs_changes
-    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'    
+    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
       fail-fast: false
       matrix:
diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml
index c60d870b05..a635e2865c 100644
--- a/.github/workflows/test_destinations.yml
+++ b/.github/workflows/test_destinations.yml
@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
 
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
diff --git a/.github/workflows/test_doc_snippets.yml b/.github/workflows/test_doc_snippets.yml
index d73c109894..7ae5c67836 100644
--- a/.github/workflows/test_doc_snippets.yml
+++ b/.github/workflows/test_doc_snippets.yml
@@ -8,6 +8,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml
index a02957b69d..11377095d0 100644
--- a/.github/workflows/test_local_destinations.yml
+++ b/.github/workflows/test_local_destinations.yml
@@ -10,6 +10,10 @@ on:
       - devel
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
diff --git a/README.md b/README.md
index 60c5c2f385..5cb681c570 100644
--- a/README.md
+++ b/README.md
@@ -41,6 +41,7 @@ Load chess game data from chess.com API and save it in DuckDB:
 ```python
 import dlt
 from dlt.sources.helpers import requests
+
 # Create a dlt pipeline that will load
 # chess player data to the DuckDB destination
 pipeline = dlt.pipeline(
@@ -48,12 +49,14 @@ pipeline = dlt.pipeline(
     destination='duckdb',
     dataset_name='player_data'
 )
+
 # Grab some player data from Chess.com API
 data = []
 for player in ['magnuscarlsen', 'rpragchess']:
     response = requests.get(f'https://api.chess.com/pub/player/{player}')
     response.raise_for_status()
     data.append(response.json())
+
 # Extract, normalize, and load the data
 pipeline.run(data, table_name='player')
 ```