diff --git a/.github/workflows/cdk_connectors_tests.yml b/.github/workflows/cdk_connectors_tests.yml new file mode 100644 index 000000000000..dba8dcd265dc --- /dev/null +++ b/.github/workflows/cdk_connectors_tests.yml @@ -0,0 +1,96 @@ +name: CDK Changes - Connectors Tests + +concurrency: + # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow. + # + # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests + # ensuring that only one run per pull request is active at a time. + # + # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches. + # This allows us to run multiple workflow dispatches in parallel. + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + workflow_dispatch: + # TODO uncomment when we will be ready to enable this + #pull_request: + # types: + # - opened + # - synchronize +jobs: + cdk_changes: + runs-on: ubuntu-latest + outputs: + python_cdk: ${{ steps.changes.outputs.python_cdk }} + permissions: + statuses: write + steps: + - name: Checkout Airbyte + if: github.event_name != 'pull_request' + uses: actions/checkout@v4 + - id: changes + uses: dorny/paths-filter@v2 + with: + filters: | + python_cdk: + - 'airbyte-cdk/python/airbyte_cdk/**' + - 'airbyte-cdk/python/bin/**' + - 'airbyte-cdk/python/poetry.lock' + - 'airbyte-cdk/python/pyproject.toml' + # The Connector CI Tests is a status check emitted by airbyte-ci + # We make it pass once we have determined that there are no changes to the connectors + - name: "Skip Connectors CI tests" + if: steps.changes.outputs.python_cdk != 'true' && github.event_name == 'pull_request' + run: | + curl --request POST \ + --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \ + --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ + --header 'content-type: application/json' \ + --data '{ + "state": "success", + "context": "CDK Changes - Connectors Tests", + "target_url": "${{ github.event.workflow_run.html_url }}" + }' \ + + connectors_ci: + needs: changes + # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR + # Forked PRs are handled by the community_ci.yml workflow + # If the condition is not met the job will be skipped (it will not fail) + if: github.event_name == 'pull_request' && needs.changes.outputs.python_cdk == 'true' && github.event.pull_request.head.repo.fork != true + name: Connectors CI + runs-on: connector-test-large + timeout-minutes: 360 # 6 hours + steps: + - name: Checkout Airbyte + uses: actions/checkout@v4 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ + ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} + - name: Fetch last commit id from remote branch [PULL REQUESTS] + if: github.event_name == 'pull_request' + id: fetch_last_commit_id_pr + run: echo "commit_id=$(git ls-remote --heads origin ${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT + - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH] + if: github.event_name == 'workflow_dispatch' + id: fetch_last_commit_id_wd + run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT + - name: Test connectors [PULL REQUESTS] + uses: ./.github/actions/run-airbyte-ci + with: + context: "pull_request" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + git_branch: ${{ github.head_ref }} + git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} + github_token: ${{ env.PAT }} + s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + # A connector test can't take more than 5 hours to run (5 * 60 * 60 = 18000 seconds) + subcommand: "connectors --execute-timeout=18000 --global-status-check-context='CDK Changes - Connectors Tests' --name source-shopify test"