Merge master to branch

.github/CODEOWNERS

@@ -5,14 +5,12 @@
 /airbyte-integrations/connectors/destination-qdrant @airbytehq/ai-language-models
 /airbyte-integrations/connectors/destination-chroma @airbytehq/ai-language-models
 /airbyte-integrations/connectors/destination-snowflake-cortex @airbytehq/ai-language-models
-/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based @airbytehq/ai-language-models
 
 # CI/CD
 /.github/ @airbytehq/dev-tooling
 /airbyte-ci/ @airbytehq/dev-tooling
 
 # Python CDK and Connector Acceptance Tests
-/airbyte-cdk/python @airbytehq/python-team
 /airbyte-integrations/connector-templates/ @airbytehq/dev-marketplace-contributions
 /airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/dev-tooling
 
@@ -23,27 +21,27 @@
 /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers
 
 # Bulk CDK
-/airbyte-cdk/bulk @airbytehq/dbsources @airbytehq/destinations
+/airbyte-cdk/bulk @airbytehq/dbsources @airbytehq/move-destinations
 /airbyte-cdk/bulk/core/extract/ @airbytehq/dbsources
-/airbyte-cdk/bulk/core/load/ @airbytehq/destinations
+/airbyte-cdk/bulk/core/load/ @airbytehq/move-destinations
 /airbyte-cdk/bulk/toolkits/extract-*/ @airbytehq/dbsources
-/airbyte-cdk/bulk/toolkits/load-*/ @airbytehq/destinations
+/airbyte-cdk/bulk/toolkits/load-*/ @airbytehq/move-destinations
 
 # Java CDK
-/airbyte-cdk/java/airbyte-cdk @airbytehq/dbsources @airbytehq/destinations
+/airbyte-cdk/java/airbyte-cdk @airbytehq/dbsources @airbytehq/move-destinations
 /airbyte-cdk/java/airbyte-cdk/*-sources/ @airbytehq/dbsources
-/airbyte-cdk/java/airbyte-cdk/*-destinations/ @airbytehq/destinations
-/airbyte-cdk/java/airbyte-cdk/typing-deduping/ @airbytehq/destinations
+/airbyte-cdk/java/airbyte-cdk/*-destinations/ @airbytehq/move-destinations
+/airbyte-cdk/java/airbyte-cdk/typing-deduping/ @airbytehq/move-destinations
 
 # Java connectors catch-all
-/buildSrc/ @airbytehq/dbsources @airbytehq/destinations
+/buildSrc/ @airbytehq/dbsources @airbytehq/move-destinations
 /airbyte-integrations/connectors/source-*/**/*.java @airbytehq/dbsources
 /airbyte-integrations/connectors/source-*/**/*.kt @airbytehq/dbsources
 /airbyte-integrations/connectors/source-*/**/*.gradle @airbytehq/dbsources
 /airbyte-integrations/connectors-performance/source-harness/ @airbytehq/dbsources
-/airbyte-integrations/connectors/destination-*/**/*.java @airbytehq/destinations
-/airbyte-integrations/connectors/destination-*/**/*.kt @airbytehq/destinations
-/airbyte-integrations/connectors/destination-*/**/*.gradle @airbytehq/destinations
+/airbyte-integrations/connectors/destination-*/**/*.java @airbytehq/move-destinations
+/airbyte-integrations/connectors/destination-*/**/*.kt @airbytehq/move-destinations
+/airbyte-integrations/connectors/destination-*/**/*.gradle @airbytehq/move-destinations
 /airbyte-integrations/connectors-performance/destination-harness/ @airbytehq/dbsources
 
 # Java-based certified or incubating source connectors
@@ -53,12 +51,12 @@
 /airbyte-integrations/connectors/source-postgres/ @airbytehq/dbsources
 
 # Java-based certified or incubating destination connectors
-/airbyte-integrations/connectors/destination-bigquery/ @airbytehq/destinations
-/airbyte-integrations/connectors/destination-postgres/ @airbytehq/destinations
-/airbyte-integrations/connectors/destination-postgres-strict-encrypt/ @airbytehq/destinations
-/airbyte-integrations/connectors/destination-s3/ @airbytehq/destinations
-/airbyte-integrations/connectors/destination-snowflake/ @airbytehq/destinations
-/airbyte-integrations/connectors/destination-redshift/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-bigquery/ @airbytehq/move-destinations
+/airbyte-integrations/connectors/destination-postgres/ @airbytehq/move-destinations
+/airbyte-integrations/connectors/destination-postgres-strict-encrypt/ @airbytehq/move-destinations
+/airbyte-integrations/connectors/destination-s3/ @airbytehq/move-destinations
+/airbyte-integrations/connectors/destination-snowflake/ @airbytehq/move-destinations
+/airbyte-integrations/connectors/destination-redshift/ @airbytehq/move-destinations
 
 # Python critical connectors
 /airbyte-integrations/connectors/source-facebook-marketing/ @airbytehq/python-team

.github/actions/run-airbyte-ci/action.yml

@@ -107,6 +107,15 @@ runs:
       id: get-start-timestamp
       shell: bash
       run: echo "start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT
+    - name: Debug-print local paths checked out
+      shell: bash
+      run: |
+        set -x
+        echo "Working directory: $(pwd)"
+        ls -la
+        ls -la airbyte-python-cdk || echo "No airbyte-python-cdk directory"
+        ls -laL ../airbyte-python-cdk || echo "No airbyte-python-cdk symlink"
+
     - name: Docker login
       id: docker-login
       uses: docker/login-action@v3

.github/workflows/airbyte-ci-tests.yml

@@ -42,7 +42,6 @@ jobs:
             - airbyte-ci/connectors/erd/**
             - airbyte-ci/connectors/metadata_service/lib/**
             - airbyte-ci/connectors/metadata_service/orchestrator/**
-            - airbyte-cdk/python/**
             - airbyte-integrations/bases/connector-acceptance-test/**
 
   run-tests:
@@ -60,6 +59,25 @@ jobs:
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.ref }}
+      - name: Checkout Airbyte Python CDK
+        uses: actions/checkout@v4
+        with:
+          repository: airbytehq/airbyte-python-cdk
+          ref: main
+          # We can't clone into a parent directory of the repo, so we clone into
+          # a subdirectory and then move it over as a sibling directory.
+          # This will be used for the `--use-local-cdk` flag in `airbyte-ci` command
+          path: airbyte-python-cdk
+      - name: Move airbyte-python-cdk to sibling directory path
+        shell: bash
+        run: mv ./airbyte-python-cdk ../airbyte-python-cdk
+      - name: Show local paths checked out
+        shell: bash
+        run: |
+          set -x
+          echo "Current directory: $(pwd)"
+          ls -la
+          ls -la ../airbyte-python-cdk || echo "No airbyte-python-cdk directory"
 
       - name: Extract branch name [WORKFLOW DISPATCH]
         shell: bash

.github/workflows/connectors_up_to_date.yml

@@ -11,22 +11,28 @@ on:
     inputs:
       connectors-options:
         description: "Options to pass to the 'airbyte-ci connectors' command group."
-        default: '--concurrency=10 --language=python --language=low-code --language=manifest-only --metadata-query="''-rc.'' not in data.dockerImageTag" --metadata-query="''source-declarative-manifest'' not in data.dockerRepository"'
-      auto-merge:
-        description: "Whether to auto-merge the PRs created by the action."
-        default: "false"
+        default: "--language=python --language=low-code --language=manifest-only"
 jobs:
-  connectors_up_to_date:
-    name: Connectors up-to-date
+  scheduled_connectors_up_to_date:
+    name: Connectors up-to-date [SCHEDULED TRIGGER]
     runs-on: connector-nightly-xlarge
+    if: github.event_name == 'schedule'
+    strategy:
+      matrix:
+        connector_language:
+          - python
+          # TODO: re-add low_code and manifest-only once new SDM image is deemed stable
+          # - low_code
+          # - manifest-only
+        support_level:
+          - certified
+          - community
     permissions:
       pull-requests: write
-    timeout-minutes: 1440 # 24 hours
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v4
       - name: Run airbyte-ci connectors up-to-date [WORKFLOW]
-        if: github.event_name == 'workflow_dispatch'
         id: airbyte-ci-connectors-up-to-date-workflow-dispatch
         uses: ./.github/actions/run-airbyte-ci
         with:
@@ -40,10 +46,19 @@ jobs:
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: "connectors ${{ github.event.inputs.connectors-options }} up-to-date  --create-prs ${{ github.event.inputs.auto-merge == 'false' && '' || '--auto-merge' }}"
-      - name: Run airbyte-ci connectors up-to-date [SCHEDULE]
-        if: github.event_name == 'schedule'
-        id: airbyte-ci-connectors-up-to-date-schedule
+          subcommand: 'connectors --concurrency=10 --language=${{ matrix.connector_language}} --support-level=${{ matrix.support_level}} --metadata-query="\"source-declarative-manifest\" not in data.dockerRepository"  --metadata-query="\"-rc.\" not in data.dockerImageTag" up-to-date  --create-prs --auto-merge'
+
+  workflow_dispatch_connectors_up_to_date:
+    name: Connectors up-to-date [MANUAL TRIGGER]
+    runs-on: connector-nightly-xlarge
+    if: github.event_name == 'workflow_dispatch'
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v4
+      - name: Run airbyte-ci connectors up-to-date [WORKFLOW]
+        id: airbyte-ci-connectors-up-to-date-workflow-dispatch
         uses: ./.github/actions/run-airbyte-ci
         with:
           context: "master"
@@ -56,4 +71,4 @@ jobs:
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: 'connectors --concurrency=10 --language=python --language=low-code --support-level=community --support-level=certified --metadata-query="\"source-declarative-manifest\" not in data.dockerRepository"  --metadata-query="\"-rc.\" not in data.dockerImageTag" up-to-date --create-prs --auto-merge'
+          subcommand: 'connectors --concurrency=10 ${{ github.event.inputs.connectors-options }} --metadata-query="\"source-declarative-manifest\" not in data.dockerRepository" --metadata-query="\"-rc.\" not in data.dockerImageTag" up-to-date  --create-prs --auto-merge'