(not tested) First draft of connector tests on CDK change workflow (#…

…40713)

.github/workflows/cdk_connectors_tests.yml

@@ -0,0 +1,96 @@
+name: CDK Changes - Connectors Tests
+
+concurrency:
+  # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow.
+  #
+  # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests
+  #  ensuring that only one run per pull request is active at a time.
+  #
+  # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches.
+  # This allows us to run multiple workflow dispatches in parallel.
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+  # TODO uncomment when we will be ready to enable this
+  #pull_request:
+  #  types:
+  #    - opened
+  #    - synchronize
+jobs:
+  cdk_changes:
+    runs-on: ubuntu-latest
+    outputs:
+      python_cdk: ${{ steps.changes.outputs.python_cdk }}
+    permissions:
+      statuses: write
+    steps:
+      - name: Checkout Airbyte
+        if: github.event_name != 'pull_request'
+        uses: actions/checkout@v4
+      - id: changes
+        uses: dorny/paths-filter@v2
+        with:
+          filters: |
+            python_cdk:
+              - 'airbyte-cdk/python/airbyte_cdk/**'
+              - 'airbyte-cdk/python/bin/**'
+              - 'airbyte-cdk/python/poetry.lock'
+              - 'airbyte-cdk/python/pyproject.toml'
+      # The Connector CI Tests is a status check emitted by airbyte-ci
+      # We make it pass once we have determined that there are no changes to the connectors
+      - name: "Skip Connectors CI tests"
+        if: steps.changes.outputs.python_cdk != 'true' && github.event_name == 'pull_request'
+        run: |
+          curl --request POST \
+          --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
+          --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
+          --header 'content-type: application/json' \
+          --data '{
+            "state": "success",
+            "context": "CDK Changes - Connectors Tests",
+            "target_url": "${{ github.event.workflow_run.html_url }}"
+            }' \
+
+  connectors_ci:
+    needs: changes
+    # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR
+    # Forked PRs are handled by the community_ci.yml workflow
+    # If the condition is not met the job will be skipped (it will not fail)
+    if: github.event_name == 'pull_request' && needs.changes.outputs.python_cdk == 'true' && github.event.pull_request.head.repo.fork != true
+    name: Connectors CI
+    runs-on: connector-test-large
+    timeout-minutes: 360 # 6 hours
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v4
+      - name: Check PAT rate limits
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
+            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
+      - name: Fetch last commit id from remote branch [PULL REQUESTS]
+        if: github.event_name == 'pull_request'
+        id: fetch_last_commit_id_pr
+        run: echo "commit_id=$(git ls-remote --heads origin ${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT
+      - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH]
+        if: github.event_name == 'workflow_dispatch'
+        id: fetch_last_commit_id_wd
+        run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT
+      - name: Test connectors [PULL REQUESTS]
+        uses: ./.github/actions/run-airbyte-ci
+        with:
+          context: "pull_request"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          git_branch: ${{ github.head_ref }}
+          git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
+          github_token: ${{ env.PAT }}
+          s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          # A connector test can't take more than 5 hours to run (5 * 60 * 60 = 18000 seconds)
+          subcommand: "connectors --execute-timeout=18000 --global-status-check-context='CDK Changes - Connectors Tests' --name source-shopify  test"