Merge remote-tracking branch 'origin/master' into xiaohan/cron-update

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.50.30
+current_version = 0.50.31
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.github/CODEOWNERS

@@ -57,3 +57,9 @@
 /airbyte-integrations/connectors/destination-s3/ @airbytehq/destinations
 /airbyte-integrations/connectors/destination-snowflake/ @airbytehq/destinations
 /airbyte-integrations/connectors/destination-tidb/ @airbytehq/destinations
+
+# Build customization file change
+/airbyte-integrations/connectors/**/build_customization.py @airbytehq/connector-operations
+
+# airbyte-ci
+/airbyte-ci @airbytehq/connector-operations

.github/workflows/airbyte-ci-tests.yml

@@ -11,37 +11,54 @@ on:
       - opened
       - reopened
       - synchronize
-    paths-ignore:
-      - "**/*.md"
 jobs:
   run-airbyte-ci-tests:
     name: Run Airbyte CI tests
     runs-on: "conn-prod-xlarge-runner"
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref }}
+
+      # IMPORTANT! This is nessesary to make sure that a status is reported on the PR
+      # even if the workflow is skipped. If we used github actions filters, the workflow
+      # would not be reported as skipped, but instead would be forever pending.
+      #
+      # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE.
+      #
+      # Also it gets worse
+      #
+      # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED.
+      # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING.
       - name: Get changed files
-        uses: dorny/paths-filter@v2
+        uses: tj-actions/changed-files@v39
         id: changes
         with:
-          filters: |
+          files_yaml: |
             ops:
               - 'airbyte-ci/connectors/connector_ops/**'
+              - '!**/*.md'
             base_images:
               - 'airbyte-ci/connectors/connector_ops/**'
               - 'airbyte-ci/connectors/base_images/**'
+              - '!**/*.md'
             pipelines:
               - 'airbyte-ci/connectors/connector_ops/**'
               - 'airbyte-ci/connectors/base_images/**'
               - 'airbyte-ci/connectors/pipelines/**'
-            metadata-lib:
+              - '!**/*.md'
+            metadata_lib:
               - 'airbyte-ci/connectors/metadata/lib/**'
-            metadata-orchestrator:
+              - '!**/*.md'
+            metadata_orchestrator:
               - 'airbyte-ci/connectors/metadata/lib/**'
               - 'airbyte-ci/connectors/metadata/orchestrator/**'
+              - '!**/*.md'
 
       - name: Run airbyte-ci/connectors/connector_ops tests
-        if: steps.changes.outputs.ops == 'true'
+        if: steps.changes.outputs.ops_any_changed == 'true'
         id: run-airbyte-ci-connectors-connector-ops-tests
         uses: ./.github/actions/run-dagger-pipeline
         with:
@@ -55,7 +72,7 @@ jobs:
 
       - name: Run airbyte-ci/connectors/pipelines tests
         id: run-airbyte-ci-connectors-pipelines-tests
-        if: steps.changes.outputs.pipelines == 'true'
+        if: steps.changes.outputs.pipelines_any_changed == 'true'
         uses: ./.github/actions/run-dagger-pipeline
         with:
           context: "pull_request"
@@ -68,7 +85,7 @@ jobs:
 
       - name: Run airbyte-ci/connectors/base_images tests
         id: run-airbyte-ci-connectors-base-images-tests
-        if: steps.changes.outputs.base_images == 'true'
+        if: steps.changes.outputs.base_images_any_changed == 'true'
         uses: ./.github/actions/run-dagger-pipeline
         with:
           context: "pull_request"
@@ -80,8 +97,8 @@ jobs:
           subcommand: "test airbyte-ci/connectors/base_images"
 
       - name: Run test pipeline for the metadata lib
-        id: metadata-lib-test-pipeline
-        if: steps.changes.outputs.metadata-lib == 'true'
+        id: metadata_lib-test-pipeline
+        if: steps.changes.outputs.metadata_lib_any_changed == 'true'
         uses: ./.github/actions/run-dagger-pipeline
         with:
           subcommand: "test airbyte-ci/connectors/metadata_service/lib/"
@@ -90,8 +107,8 @@ jobs:
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
       - name: Run test for the metadata orchestrator
-        id: metadata-orchestrator-test-pipeline
-        if: steps.changes.outputs.metadata-orchestrator == 'true'
+        id: metadata_orchestrator-test-pipeline
+        if: steps.changes.outputs.metadata_orchestrator_any_changed == 'true'
         uses: ./.github/actions/run-dagger-pipeline
         with:
           subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/"

.github/workflows/connector-performance-command.yml

@@ -41,6 +41,7 @@ on:
         options:
           - connectors/source-postgres
           - connectors/source-mysql
+          - connectors/source-mongodb-v2
           - connectors/destination-snowflake
         default: "connectors/source-postgres"
       repo:
@@ -68,7 +69,15 @@ on:
       sync-mode:
         description: "Sync mode to use for destination performance measurement."
         required: false
+        type: choice
+        options:
+          - full_refresh
+          - incremental
         default: "full_refresh"
+      reportToDatadog:
+        description: "Whether to report the performance test results to Datadog."
+        required: false
+        default: "false"
 jobs:
   uuid:
     name: "Custom UUID of workflow run"
@@ -132,8 +141,9 @@ jobs:
       - name: Filter supported connectors
         if: "${{ inputs.connector != 'connectors/source-postgres' &&
           inputs.connector != 'connectors/source-mysql' &&
-          inputs.connector != 'connectors/destination-snowflake' }}"
-        run: echo "Only connectors/source-postgres, source-mysql and destination-snowflake currently supported by harness" && exit 1
+          inputs.connector != 'connectors/destination-snowflake' &&
+          inputs.connector != 'connectors/source-mongodb-v2' }}"
+        run: echo "Only connectors/source-postgres, source-mysql, source-mongodb-v2 and destination-snowflake currently supported by harness" && exit 1
       - name: Checkout Airbyte
         uses: actions/checkout@v3
         with:
@@ -197,6 +207,7 @@ jobs:
           DS: ${{ inputs.dataset }}
           STREAM_NUMBER: ${{ inputs.stream-number }}
           SYNC_MODE: ${{ inputs.sync-mode }}
+          REPORT_TO_DATADOG: ${{ inputs.reportToDatadog }}
           PREFIX: '{"type":"LOG","log":{"level":"INFO","message":"INFO i.a.i.p.PerformanceTest(runTest):165'
           SUFFIX: '"}}'
           HARNESS_TYPE: ${{ steps.which-harness.outputs.harness_type }}
@@ -206,13 +217,10 @@ jobs:
           connector_name=$(echo $CONN | cut -d / -f 2)
           kind load docker-image airbyte/$connector_name:dev --name chart-testing
           kind load docker-image airbyte/$HARNESS_TYPE:dev --name chart-testing
-          # envsubst requires variables to be exported
+          # envsubst requires variables to be exported or setup in the env field in this step.
           export CONNECTOR_IMAGE_NAME=${CONN/connectors/airbyte}:dev
           export DATASET=$DS
-          export STREAM_NUMBER=$STREAM_NUMBER
-          export SYNC_MODE=$SYNC_MODE
           export HARNESS=$HARNESS_TYPE
-          export DD_API_KEY=$DD_API_KEY
           envsubst < ./tools/bin/run-harness-process.yaml | kubectl create -f -
           echo "harness is ${{ steps.which-harness.outputs.harness_type }}"
           POD=$(kubectl get pod -l app=performance-harness -o jsonpath="{.items[0].metadata.name}")

.github/workflows/connector_teams_review_requirements.yml

@@ -2,6 +2,11 @@ name: Connector Ops CI - Check review requirements
 
 on:
   pull_request:
+    types:
+      - opened
+      - reopened
+      - ready_for_review
+      - synchronize
     paths:
       - "airbyte-integrations/connectors/source-**"
   pull_request_review:
@@ -12,7 +17,7 @@ jobs:
     name: "Check if a review is required from Connector teams"
     runs-on: ubuntu-latest
 
-    if: ${{ github.repository == 'airbytehq/airbyte' }}
+    if: ${{ github.repository == 'airbytehq/airbyte' && github.event.pull_request.draft == false }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
@@ -39,4 +44,5 @@ jobs:
         with:
           status: ${{ steps.get-mandatory-reviewers.outputs.MANDATORY_REVIEWERS  }}
           token: ${{ secrets.OCTAVIA_4_ROOT_ACCESS }}
+          request-reviews: true
           requirements-file: .github/connector_org_review_requirements.yaml

.github/workflows/format.yml

@@ -13,7 +13,7 @@ jobs:
   format-and-commit:
     runs-on: ubuntu-latest
     name: "Apply All Formatting Rules"
-    timeout-minutes: 20
+    timeout-minutes: 40
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
@@ -23,6 +23,25 @@ jobs:
           # Without this we would be be forever waiting on required checks to pass
           token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
 
+      # IMPORTANT! This is nessesary to make sure that a status is reported on the PR
+      # even if the workflow is skipped. If we used github actions filters, the workflow
+      # would not be reported as skipped, but instead would be forever pending.
+      #
+      # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE.
+      #
+      # Also it gets worse
+      #
+      # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED.
+      # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING.
+      - name: Get changed files
+        uses: tj-actions/changed-files@v39
+        id: changes
+        with:
+          files_yaml: |
+            format:
+              - '**/*'
+              - '!**/*.md'
+
       - uses: actions/setup-java@v3
         with:
           distribution: "zulu"
@@ -47,6 +66,7 @@ jobs:
           EOF
 
       - name: Format
+        if: steps.changes.outputs.format_any_changed == 'true'
         uses: Wandalen/[email protected]
         with:
           command: ./gradlew format --scan --info --stacktrace

.github/workflows/gradle.yml

@@ -59,15 +59,38 @@ jobs:
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
+
+      # IMPORTANT! This is nessesary to make sure that a status is reported on the PR
+      # even if the workflow is skipped. If we used github actions filters, the workflow
+      # would not be reported as skipped, but instead would be forever pending.
+      #
+      # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE.
+      #
+      # Also it gets worse
+      #
+      # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED.
+      # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING.
+      - name: Get changed files
+        uses: tj-actions/changed-files@v39
+        id: changes
+        with:
+          files_yaml: |
+            gradlecheck:
+              - '**/*'
+              - '!**/*.md'
+
       - uses: actions/setup-java@v3
         with:
           distribution: "zulu"
           java-version: "17"
       - name: Install Pip
+        if: steps.changes.outputs.gradlecheck_any_changed == 'true'
         run: curl -fsSL https://bootstrap.pypa.io/get-pip.py | python3
       - name: Install Pyenv
+        if: steps.changes.outputs.gradlecheck_any_changed == 'true'
         run: python3 -m pip install virtualenv --user
       - name: Run Gradle Check
+        if: steps.changes.outputs.gradlecheck_any_changed == 'true'
         uses: burrunan/gradle-cache-action@v1
         env:
           CI: true

.github/workflows/run-mypy-on-modified-cdk-files.yml

@@ -22,9 +22,7 @@ jobs:
         id: changed-py-files
         uses: tj-actions/changed-files@v23
         with:
-          files: |
-            *.py
-            **/*.py
+          files: "airbyte-cdk/python/airbyte_cdk/**/*.py"
       - name: Run if any of the listed files above is changed
         if: steps.changed-py-files.outputs.any_changed == 'true'
         run: mypy ${{ steps.changed-py-files.outputs.all_changed_files }} --config-file airbyte-cdk/python/mypy.ini --install-types --non-interactive

.github/workflows/tmp-source-postgres-test.yml

@@ -0,0 +1,43 @@
+# This workflows runs airbyte-ci connectors --name=source-postgres test
+# We created this in the context of our project to improve CI performances for this connector
+# It's made to be triggered manually from the GitHub UI
+# It will allow us to collect performance metrics outside of the context of nightly builds
+# And also to use different runner types (e.g. conn-prod-xxlarge-runner) to test the connector with various resources.
+
+name: source-postgres ci - for testing only
+
+on:
+  schedule:
+    # Run three time a day to collect performance metrics and observe variance
+    - cron: "0 9,13,17 * * *"
+  workflow_dispatch:
+    inputs:
+      runner:
+        description: "The runner to use for this job"
+        default: "conn-prod-xlarge-runner"
+
+jobs:
+  source_postgres_ci:
+    name: Source Postgres CI on ${{ inputs.runner || 'conn-prod-xlarge-runner'}}
+    runs-on: ${{ inputs.runner || 'conn-prod-xlarge-runner'}}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ github.event.inputs.repo }}
+          ref: ${{ github.event.inputs.gitref }}
+      - name: Extract branch name
+        shell: bash
+        run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
+        id: extract_branch
+      - name: Test source-postgres
+        uses: ./.github/actions/run-dagger-pipeline
+        with:
+          context: "master"
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          git_branch: ${{ steps.extract_branch.outputs.branch }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          subcommand: "connectors --name=source-postgres test"

airbyte-cdk/java/airbyte-cdk/README.md

@@ -156,6 +156,7 @@ MavenLocal debugging steps:
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                        |
 |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.1.11  | 2023-10-18 | [\#31486](https://github.com/airbytehq/airbyte/pull/31486) | Update constants in AdaptiveSourceRunner.                                                                                                                      |
 | 0.1.9   | 2023-10-12 | [\#31309](https://github.com/airbytehq/airbyte/pull/31309) | Use toPlainString() when handling BigDecimals in PostgresConverter                                                                                             |
 | 0.1.8   | 2023-10-11 | [\#31322](https://github.com/airbytehq/airbyte/pull/31322) | Cap log line length to 32KB to prevent loss of records                                                                                                         |
 | 0.1.7   | 2023-10-10 | [\#31194](https://github.com/airbytehq/airbyte/pull/31194) | Deallocate unused per stream buffer memory when empty                                                                                                          |

airbyte-cdk/java/airbyte-cdk/build.gradle

@@ -30,8 +30,8 @@ subprojects { subproject ->
         }
         repositories {
             maven {
-                name 'cloudrepo'
-                url 'https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars'
+                name 'airbyte-public-jars'
+                url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/'
                 credentials {
                     username System.getenv('CLOUDREPO_USER')
                     password System.getenv('CLOUDREPO_PASSWORD')

airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveSourceRunner.java

@@ -17,8 +17,8 @@ public class AdaptiveSourceRunner {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(AdaptiveSourceRunner.class);
 
-  private static final String DEPLOYMENT_MODE_KEY = "DEPLOYMENT_MODE";
-  private static final String COULD_MODE = "CLOUD";
+  public static final String DEPLOYMENT_MODE_KEY = "DEPLOYMENT_MODE";
+  public static final String CLOUD_MODE = "CLOUD";
 
   public static OssSourceBuilder baseOnEnv() {
     final String mode = System.getenv(DEPLOYMENT_MODE_KEY);
@@ -71,7 +71,7 @@ public Runner(final String deploymentMode,
 
     private Source getSource() {
       LOGGER.info("Running source under deployment mode: {}", deploymentMode);
-      if (deploymentMode != null && deploymentMode.equals(COULD_MODE)) {
+      if (deploymentMode != null && deploymentMode.equals(CLOUD_MODE)) {
         return cloudSourceSupplier.get();
       }
       if (deploymentMode == null) {

airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/NamingConventionTransformer.java

@@ -51,6 +51,10 @@ public interface NamingConventionTransformer {
   @Deprecated
   String getTmpTableName(String name);
 
+  default String getTmpTableName(final String streamName, final String randomSuffix) {
+    return getTmpTableName(streamName);
+  }
+
   String convertStreamName(final String input);
 
   String applyDefaultCase(final String input);