Merge branch 'master' into mihaly/extend-metrics-for-sponsored-brands

.github/CODEOWNERS

@@ -7,15 +7,15 @@
 /airbyte-integrations/connectors/destination-snowflake-cortex @airbytehq/ai-language-models
 
 # CI/CD
-/.github/ @airbytehq/dev-tooling
-/airbyte-ci/ @airbytehq/dev-tooling
+/.github/ @airbytehq/dev-extensibility
+/airbyte-ci/ @airbytehq/dev-extensibility
 
 # Python CDK and Connector Acceptance Tests
-/airbyte-integrations/connector-templates/ @airbytehq/dev-marketplace-contributions
-/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/dev-tooling
+/airbyte-integrations/connector-templates/ @airbytehq/dev-extensibility
+/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/dev-extensibility
 
 # Build customization file change
-/airbyte-integrations/connectors/**/build_customization.py @airbytehq/dev-tooling
+/airbyte-integrations/connectors/**/build_customization.py @airbytehq/dev-extensibility
 
 # Protocol related items
 /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers
@@ -59,8 +59,8 @@
 /airbyte-integrations/connectors/destination-redshift/ @airbytehq/move-destinations
 
 # Python critical connectors
-/airbyte-integrations/connectors/source-facebook-marketing/ @airbytehq/python-team
-/airbyte-integrations/connectors/source-hubspot/ @airbytehq/python-team
-/airbyte-integrations/connectors/source-salesforce/ @airbytehq/python-team
-/airbyte-integrations/connectors/source-shopify/ @airbytehq/python-team
-/airbyte-integrations/connectors/source-stripe/ @airbytehq/python-team
+/airbyte-integrations/connectors/source-facebook-marketing/ @airbytehq/dev-extensibility
+/airbyte-integrations/connectors/source-hubspot/ @airbytehq/dev-extensibility
+/airbyte-integrations/connectors/source-salesforce/ @airbytehq/dev-extensibility
+/airbyte-integrations/connectors/source-shopify/ @airbytehq/dev-extensibility
+/airbyte-integrations/connectors/source-stripe/ @airbytehq/dev-extensibility

.github/ISSUE_TEMPLATE/8-documentation.yaml

@@ -1,16 +1,17 @@
-name: 📚 Request or report a problem with documenation
-description: For all docs typos, tutorials requests, etc.
-labels: [type/enhancement, area/documentation needs-triage]
+name: 📚 Request or report a problem with documentation
+description: Typos, tutorial requests, and everything in between.
+title: "[Docs]: "
+labels: [area/documentation, needs-triage]
 body:
   - type: input
     id: url
     attributes:
       label: Page URL
-      description: Paste the URL with problem when reporting an issue
+      description: If this is an issue with a specific page, paste the URL here.
       placeholder: https://docs.airbyte.com/connector-development/connector-specification-reference
   - type: textarea
     id: description
     attributes:
       label: Description
       description: >-
-        It can be as pedantic as a typo or as titanic as an entire tutorial request.
+        Please provide as much detail as possible to ensure we can resolve the issue properly.

.github/actions/install-airbyte-ci/action.yml

@@ -56,13 +56,16 @@ runs:
         sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
         sudo chmod +x /usr/local/bin/airbyte-ci
 
-    - name: Install Python 3.10
-      id: install-python-3-10
+    - name: Install Python 3.11
+      id: install-python-3-11
       uses: actions/setup-python@v4
       # if: steps.determine-install-mode.outputs.install-mode == 'source'
       with:
-        python-version: "3.10"
+        python-version: "3.11"
         token: ${{ inputs.github_token }}
+        cache: "pip"
+        check-latest: true
+        update-environment: true
 
     - name: Install Airbyte CI from source
       id: install-airbyte-ci-source

.github/actions/install-java-environment/action.yml

@@ -0,0 +1,29 @@
+name: Install Java Environment
+description: "Installs the Java environment"
+inputs:
+  java_version:
+    description: "Java version"
+    required: false
+    default: "21"
+    type: string
+  gradle_cache_read_only:
+    description: "Whether to use a read-only Gradle cache"
+    required: false
+    default: false
+    type: boolean
+  gradle_cache_write_only:
+    description: "Whether to use a write-only Gradle cache"
+    required: false
+    default: false
+    type: boolean
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-java@v4
+      with:
+        distribution: corretto
+        java-version: ${{ inputs.java_version }}
+    - uses: gradle/actions/setup-gradle@v3
+      with:
+        cache-read-only: ${{ inputs.gradle_cache_read_only }}
+        cache-write-only: ${{ inputs.gradle_cache_write_only }}

.github/actions/run-airbyte-ci/action.yml

@@ -115,7 +115,9 @@ runs:
         ls -la
         ls -la airbyte-python-cdk || echo "No airbyte-python-cdk directory"
         ls -laL ../airbyte-python-cdk || echo "No airbyte-python-cdk symlink"
-
+    - name: Install Java Environment
+      id: install-java-environment
+      uses: ./.github/actions/install-java-environment
     - name: Docker login
       id: docker-login
       uses: docker/login-action@v3

.github/workflows/airbyte-ci-release.yml

@@ -38,7 +38,9 @@ jobs:
         id: install_python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
+          check-latest: true
+          update-environment: true
 
       - name: Install Poetry
         id: install_poetry

.github/workflows/airbyte-ci-tests.yml

@@ -34,7 +34,6 @@ jobs:
             internal_poetry_packages:
             - airbyte-ci/connectors/pipelines/**
             - airbyte-ci/connectors/base_images/**
-            - airbyte-ci/connectors/common_utils/**
             - airbyte-ci/connectors/connectors_insights/**
             - airbyte-ci/connectors/connector_ops/**
             - airbyte-ci/connectors/connectors_qa/**

.github/workflows/auto_merge.yml

@@ -15,7 +15,9 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
+          check-latest: true
+          update-environment: true
       - name: Install and configure Poetry
         uses: snok/install-poetry@v1
         with:

.github/workflows/bump-version-command.yml

@@ -17,10 +17,10 @@ on:
         default: "patch"
 
       changelog:
-        description: "The comment to add to the changelog."
+        description: "Optional. The comment to add to the changelog. If not provided, the PR title will be used."
         required: false
-        # TODO: We could infer the changelog string from the PR description!
-        default: "Bumped automatically in the pull request, please see PR description"
+        # If not provided, we'll use the PR title as the changelog
+        default: ""
 
       # These must be declared, but they are unused and ignored.
       # TODO: Infer 'repo' and 'gitref' from PR number on other workflows, so we can remove these.
@@ -52,6 +52,7 @@ jobs:
           PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr }})
           echo "repo=$(echo "$PR_JSON" | jq -r .head.repo.full_name)" >> $GITHUB_OUTPUT
           echo "branch=$(echo "$PR_JSON" | jq -r .head.ref)" >> $GITHUB_OUTPUT
+          echo "pr_title=$(echo "$PR_JSON" | jq -r .title)" >> $GITHUB_OUTPUT
           echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
 
       - name: Checkout Airbyte
@@ -78,6 +79,14 @@ jobs:
 
             [1]: ${{ steps.job-vars.outputs.run-url }}
 
+      - name: Log changelog source
+        run: |
+          if [ -n "${{ github.event.inputs.changelog }}" ]; then
+            echo "Using user-provided changelog: ${{ github.event.inputs.changelog }}"
+          else
+            echo "Using PR title as changelog: ${{ steps.job-vars.outputs.pr_title }}"
+          fi
+
       - name: Run airbyte-ci connectors --modified bump-version
         uses: ./.github/actions/run-airbyte-ci
         continue-on-error: true
@@ -86,10 +95,11 @@ jobs:
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          git_repo_url: https://github.com/${{ steps.job-vars.outputs.repo }}.git
           subcommand: |
             connectors --modified bump-version \
               ${{ github.event.inputs.type }} \
-              "${{ github.event.inputs.changelog }}" \
+              "${{ github.event.inputs.changelog != '' && github.event.inputs.changelog || steps.job-vars.outputs.pr_title }}" \
               --pr-number ${{ github.event.inputs.pr }}
 
       # This is helpful in the case that we change a previously committed generated file to be ignored by git.

.github/workflows/connector-performance-command.yml

@@ -168,7 +168,9 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
+          check-latest: true
+          update-environment: true
       - name: Install CI scripts
         run: |
           pip install pipx

.github/workflows/connectors_nightly_build.yml

@@ -5,18 +5,38 @@ on:
     # 0AM UTC is 2AM CEST, 3AM EEST, 5PM PDT.
     - cron: "0 0 * * *"
   workflow_dispatch:
-    inputs:
-      test-connectors-options:
-        default: --concurrency=5 --support-level=certified
-        required: true
 
 run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }}"
 
 jobs:
+  generate_matrix:
+    name: Generate matrix
+    runs-on: ubuntu-24.04
+    outputs:
+      generated_matrix: ${{ steps.generate_matrix.outputs.generated_matrix }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v4
+      - name: Run airbyte-ci connectors list [SCHEDULED TRIGGER]
+        id: airbyte-ci-connectors-list-scheduled
+        uses: ./.github/actions/run-airbyte-ci
+        with:
+          context: "master"
+          subcommand: "connectors --support-level=certified list --output=selected_connectors.json"
+      - name: Generate matrix - 30 connectors per job
+        id: generate_matrix
+        run: |
+          matrix=$(jq -c -r '{include: [.[] | "--name=" + .] | to_entries | group_by(.key / 30 | floor) | map(map(.value) | {"connector_names": join(" ")})}' selected_connectors.json)
+          echo "generated_matrix=$matrix" >> $GITHUB_OUTPUT
+
   test_connectors:
+    needs: generate_matrix
     name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }}"
-    timeout-minutes: 720 # 12 hours
     runs-on: connector-nightly-xlarge
+    continue-on-error: true
+    strategy:
+      matrix: ${{fromJson(needs.generate_matrix.outputs.generated_matrix)}}
+
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
@@ -32,7 +52,7 @@ jobs:
         with:
           context: "master"
           ci_job_key: "nightly_builds"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_3 }}
+          # dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_3 }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
@@ -41,4 +61,4 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: "connectors ${{ inputs.test-connectors-options || '--concurrency=8 --support-level=certified' }} test"
+          subcommand: "connectors ${{ matrix.connector_names}} test"

.github/workflows/connectors_version_increment_check.yml → .github/workflows/connectors_pre_release_checks.yml

@@ -1,4 +1,4 @@
-name: Connectors Version Increment Check
+name: Connectors Pre-Release Checks
 
 concurrency:
   # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow.
@@ -20,9 +20,11 @@ on:
       - "airbyte-integrations/connectors/**/*"
 jobs:
   connectors_ci:
-    name: Connectors Version Increment Check
+    name: Connector Pre-Release Checks
     runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings
-    if: github.event.pull_request.head.repo.fork != true
+    if: >
+      github.event.pull_request.head.repo.fork != true &&
+      github.event.pull_request.draft == false
     timeout-minutes: 22
     steps:
       - name: Checkout Airbyte
@@ -51,4 +53,4 @@ jobs:
           github_token: ${{ env.PAT }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: "connectors --modified test --only-step=version_inc_check --global-status-check-context='Version increment check for Java connectors' --global-status-check-description='Checking if java connectors modified in this PR got their version bumped'"
+          subcommand: "connectors --modified test --only-step=version_inc_check --only-step=qa_checks --global-status-check-context='Connectors Pre-Release Check' --global-status-check-description='Checking if connectors modified in this PR are ready for release'"

.github/workflows/connectors_tests.yml

@@ -133,4 +133,4 @@ jobs:
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           # A connector test can't take more than 5 hours to run (5 * 60 * 60 = 18000 seconds)
-          subcommand: "connectors --execute-timeout=18000 --modified  test"
+          subcommand: "connectors --execute-timeout=18000 --modified test --skip-step=qa_checks --skip-step=version_inc_check"