Merge branch 'master' into akash/lsn-investigate

.github/ISSUE_TEMPLATE/config.yml

@@ -2,5 +2,5 @@
 blank_issues_enabled: false
 contact_links:
   - name: Ask a question, get community support or request new features/connectors
-    url: https://github.com/apache/airflow/discussions/
+    url: https://github.com/airbytehq/airbyte/discussions/
     about: Use Github Discussion to request features/connectors or discuss ideas or issues.

.github/workflows/format_check.yml

@@ -1,27 +1,27 @@
-name: Check for formatting errors on head ref
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref }}
-
+name: Check for formatting errors
+run-name: Check for formatting errors on ${{ github.ref }}
 on:
   workflow_dispatch:
   push:
     branches:
       - master
+  pull_request:
+
 jobs:
   format-check:
     runs-on: "ci-runner-connector-format-medium-dagger-0-6-4"
-    name: "Check for formatting errors on ${{ github.head_ref }}"
-    timeout-minutes: 40
+    # IMPORTANT: This name must match the require check name on the branch protection settings
+    name: "Check for formatting errors"
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
         with:
           ref: ${{ github.head_ref }}
           token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
 
-      - name: Run airbyte-ci format check
-        id: airbyte_ci_format_check_all
+      - name: Run airbyte-ci format check [MASTER]
+        id: airbyte_ci_format_check_all_master
+        if: github.ref == 'refs/heads/master'
         uses: ./.github/actions/run-dagger-pipeline
         continue-on-error: true
         with:
@@ -34,18 +34,46 @@ jobs:
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
           subcommand: "format check all"
 
-      # This is helpful in the case that we change a previously committed generated file to be ignored by git.
-      - name: Remove any files that have been gitignored
-        run: git ls-files -i -c --exclude-from=.gitignore | xargs -r git rm --cached
+      - name: Run airbyte-ci format check [PULL REQUEST]
+        id: airbyte_ci_format_check_all_pr
+        if: github.event_name == 'pull_request'
+        uses: ./.github/actions/run-dagger-pipeline
+        continue-on-error: false
+        with:
+          context: "pull_request"
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
+          subcommand: "format check all"
 
-      - name: Match GitHub User to Slack User
+      - name: Run airbyte-ci format check [WORKFLOW DISPATCH]
+        id: airbyte_ci_format_check_all_manual
+        if: github.event_name == 'workflow_dispatch'
+        uses: ./.github/actions/run-dagger-pipeline
+        continue-on-error: false
+        with:
+          context: "manual"
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
+          subcommand: "format check all"
+
+      - name: Match GitHub User to Slack User [MASTER]
+        if: github.ref == 'refs/heads/master'
         id: match-github-to-slack-user
         uses: ./.github/actions/match-github-to-slack-user
         env:
           AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }}
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Format Failure on Master Slack Channel
-        if: steps.airbyte_ci_format_check_all.outcome == 'failure'
+
+      - name: Format Failure on Master Slack Channel [MASTER]
+        if: steps.airbyte_ci_format_check_all.outcome == 'failure' && github.ref == 'refs/heads/master'
         uses: abinoda/slack-action@master
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}

.github/workflows/format_fix.yml

@@ -1,4 +1,5 @@
-name: Automatic Formatting on PRs
+name: Fix formatting on a branch
+run-name: Fix formatting on ${{ github.ref }}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -7,44 +8,24 @@ concurrency:
 
 on:
   workflow_dispatch:
-  pull_request:
 jobs:
   format-fix:
     runs-on: "ci-runner-connector-format-medium-dagger-0-6-4"
-    name: "Apply All Formatting Rules"
-    timeout-minutes: 40
+    name: "Run airbyte-ci format fix all"
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
         with:
-          ref: ${{ github.head_ref }}
+          ref: ${{ github.ref }}
           # Important that this is set so that CI checks are triggered again
           # Without this we would be forever waiting on required checks to pass
           token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
 
-      # IMPORTANT! This is necessary to make sure that a status is reported on the PR
-      # even if the workflow is skipped. If we used GitHub Actions filters, the workflow
-      # would not be reported as skipped, but instead would be forever pending.
-      #
-      # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE.
-      #
-      # Also, it gets worse
-      #
-      # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED.
-      # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING.
-      - name: Get changed files
-        uses: tj-actions/changed-files@v39
-        id: changes
-        with:
-          files_yaml: |
-            format:
-              - '**/*'
-              - '!**/*.md'
-
       - name: Run airbyte-ci format fix all
         uses: ./.github/actions/run-dagger-pipeline
+        continue-on-error: true
         with:
-          context: "pull_request"
+          context: "manual"
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
@@ -55,27 +36,13 @@ jobs:
 
       # This is helpful in the case that we change a previously committed generated file to be ignored by git.
       - name: Remove any files that have been gitignored
-        if: always()
         run: git ls-files -i -c --exclude-from=.gitignore | xargs -r git rm --cached
 
       - name: Commit Formatting Changes (PR)
         uses: stefanzweifel/git-auto-commit-action@v5
-        # do not commit if master branch
-        if: github.ref != 'refs/heads/master' && always()
+        # Don't commit if we're on master
+        if: github.ref != 'refs/heads/master'
         with:
-          commit_message: Automated Commit - Formatting Changes
+          commit_message: "chore: format code"
           commit_user_name: Octavia Squidington III
           commit_user_email: [email protected]
-
-      - name: Run airbyte-ci format check all
-        if: always()
-        uses: ./.github/actions/run-dagger-pipeline
-        with:
-          context: "pull_request"
-          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
-          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
-          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
-          tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
-          subcommand: "format check all"

.pre-commit-config.yaml

@@ -8,4 +8,3 @@ repos:
         name: Run airbyte-ci format fix on git push (~30s)
         pass_filenames: false
         stages: [push]
-        verbose: true

airbyte-cdk/python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.56.1
+current_version = 0.57.1
 commit = False
 
 [bumpversion:file:setup.py]

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.57.1
+Improve integration tests tooling
+
+## 0.57.0
+low-code: cache requests sent for parent streams
+
 ## 0.56.1
 no-op to verify pypi publish flow
 

airbyte-cdk/python/Dockerfile

@@ -10,7 +10,7 @@ RUN apk --no-cache upgrade \
     && apk --no-cache add tzdata build-base
 
 # install airbyte-cdk
-RUN pip install --prefix=/install airbyte-cdk==0.56.1
+RUN pip install --prefix=/install airbyte-cdk==0.57.1
 
 # build a clean environment
 FROM base
@@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 # needs to be the same as CDK
-LABEL io.airbyte.version=0.56.1
+LABEL io.airbyte.version=0.57.1
 LABEL io.airbyte.name=airbyte/source-declarative-manifest

airbyte-cdk/python/airbyte_cdk/exception_handler.py

@@ -4,17 +4,25 @@
 
 import logging
 import sys
+from types import TracebackType
+from typing import Any, Optional
 
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 
+def assemble_uncaught_exception(exception_type: type[BaseException], exception_value: BaseException) -> AirbyteTracedException:
+    if issubclass(exception_type, AirbyteTracedException):
+        return exception_value  # type: ignore  # validated as part of the previous line
+    return AirbyteTracedException.from_exception(exception_value)
+
+
 def init_uncaught_exception_handler(logger: logging.Logger) -> None:
     """
     Handles uncaught exceptions by emitting an AirbyteTraceMessage and making sure they are not
     printed to the console without having secrets removed.
     """
 
-    def hook_fn(exception_type, exception_value, traceback_):
+    def hook_fn(exception_type: type[BaseException], exception_value: BaseException, traceback_: Optional[TracebackType]) -> Any:
         # For developer ergonomics, we want to see the stack trace in the logs when we do a ctrl-c
         if issubclass(exception_type, KeyboardInterrupt):
             sys.__excepthook__(exception_type, exception_value, traceback_)
@@ -23,11 +31,7 @@ def hook_fn(exception_type, exception_value, traceback_):
         logger.fatal(exception_value, exc_info=exception_value)
 
         # emit an AirbyteTraceMessage for any exception that gets to this spot
-        traced_exc = (
-            exception_value
-            if issubclass(exception_type, AirbyteTracedException)
-            else AirbyteTracedException.from_exception(exception_value)
-        )
+        traced_exc = assemble_uncaught_exception(exception_type, exception_value)
 
         traced_exc.emit_message()
 

airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1244,6 +1244,11 @@ definitions:
           - query: 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
           - searchIn: "{{ ','.join(config.get('search_in', [])) }}"
           - sort_by[asc]: updated_at
+      use_cache:
+        title: Use Cache
+        description: Enables stream requests caching. This field is automatically set by the CDK.
+        type: boolean
+        default: false
       $parameters:
         type: object
         additionalProperties: true