Merge master to branch

.github/workflows/community_ci.yml

@@ -172,7 +172,6 @@ jobs:
         uses: ./.github/actions/run-airbyte-ci
         with:
           context: "pull_request"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_4 }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
@@ -237,7 +236,6 @@ jobs:
         uses: ./.github/actions/run-airbyte-ci
         with:
           context: "pull_request"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_4 }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}

.github/workflows/connectors_up_to_date.yml

@@ -56,4 +56,4 @@ jobs:
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: 'connectors --concurrency=10 --language=python --language=low-code --support-level=community --support-level=certified --metadata-query="\"source-declarative-manifest\" not in data.dockerRepository"  --metadata-query="\"-rc.\" not in data.dockerImageTag" up-to-date --ignore-connector=source-declarative-manifest --create-prs --auto-merge'
+          subcommand: 'connectors --concurrency=10 --language=python --language=low-code --support-level=community --support-level=certified --metadata-query="\"source-declarative-manifest\" not in data.dockerRepository"  --metadata-query="\"-rc.\" not in data.dockerImageTag" up-to-date --create-prs --auto-merge'

.github/workflows/connectors_version_increment_check.yml

@@ -23,7 +23,7 @@ jobs:
     name: Connectors Version Increment Check
     runs-on: connector-test-large
     if: github.event.pull_request.head.repo.fork != true
-    timeout-minutes: 12
+    timeout-minutes: 22
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v4

.github/workflows/live_tests.yml

@@ -116,7 +116,7 @@ jobs:
         uses: ./.github/actions/run-airbyte-ci
         with:
           context: "manual"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_4 }}
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_3 }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}

.github/workflows/publish-bulk-cdk.yml

@@ -76,17 +76,6 @@ jobs:
           gradle-distribution-sha-256-sum-warning: false
           arguments: --scan :airbyte-cdk:bulk:bulkCdkBuild
 
-      - name: Integration test Bulk CDK
-        uses: burrunan/gradle-cache-action@v1
-        env:
-          CI: true
-        with:
-          read-only: true
-          job-id: bulk-cdk-publish
-          concurrent: true
-          gradle-distribution-sha-256-sum-warning: false
-          arguments: --scan :airbyte-cdk:bulk:bulkCdkIntegrationTest
-
       - name: Publish Poms and Jars to CloudRepo
         uses: burrunan/gradle-cache-action@v1
         env:

.github/workflows/publish_connectors.yml

@@ -30,7 +30,7 @@ jobs:
         uses: ./.github/actions/run-airbyte-ci
         with:
           context: "master"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_3 }}
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_4 }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
@@ -53,7 +53,7 @@ jobs:
         uses: ./.github/actions/run-airbyte-ci
         with:
           context: "manual"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_3 }}
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_4 }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}

.github/workflows/python-cdk-pdoc-generate.yml

@@ -0,0 +1,37 @@
+name: "Python CDK: Generate Docs"
+
+on:
+  push:
+    branches:
+      - main
+  pull_request: {}
+
+jobs:
+  preview_docs:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up Poetry
+        uses: Gr1N/setup-poetry@v9
+        with:
+          poetry-version: "1.7.1"
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "poetry"
+
+      - name: Install dependencies
+        run: cd airbyte-cdk/python && poetry install --all-extras
+
+      - name: Generate documentation
+        run: |
+          cd airbyte-cdk/python && poetry run poe docs-generate
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: "docs-generated"
+          path: "airbyte-cdk/python/docs/generated"

.github/workflows/regression_tests.yml

@@ -116,7 +116,7 @@ jobs:
         uses: ./.github/actions/run-airbyte-ci
         with:
           context: "manual"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_4 }}
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_3 }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}

airbyte-cdk/bulk/build.gradle

@@ -61,10 +61,6 @@ allprojects {
     }
 }
 
-tasks.register('bulkCdkIntegrationTest').configure {
-    dependsOn allprojects.collect {it.tasks.matching { it.name == 'integrationTest' }}
-}
-
 if (buildNumberFile.exists()) {
     tasks.register('bulkCdkBuild').configure {
         dependsOn allprojects.collect {it.tasks.named('build')}

airbyte-cdk/bulk/core/base/build.gradle

@@ -50,3 +50,7 @@ dependencies {
     testFixturesApi 'io.micronaut.test:micronaut-test-junit5:4.5.0'
     testFixturesApi 'io.github.deblockt:json-diff:1.1.0'
 }
+
+test {
+    environment "PRESENT", "present-value"
+}

airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/command/SshTunnelConfiguration.kt

@@ -10,6 +10,6 @@ import io.airbyte.cdk.ssh.SshTunnelMethodConfiguration
 interface SshTunnelConfiguration {
     val realHost: String
     val realPort: Int
-    val sshTunnel: SshTunnelMethodConfiguration
+    val sshTunnel: SshTunnelMethodConfiguration?
     val sshConnectionOptions: SshConnectionOptions
 }

airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/ssh/TunnelSession.kt

@@ -48,7 +48,7 @@ internal constructor(
 /** Creates an open [TunnelSession]. */
 fun createTunnelSession(
     remote: SshdSocketAddress,
-    sshTunnel: SshTunnelMethodConfiguration,
+    sshTunnel: SshTunnelMethodConfiguration?,
     connectionOptions: SshConnectionOptions,
 ): TunnelSession {
     if (sshTunnel is SshNoTunnelMethod) {
@@ -62,7 +62,8 @@ fun createTunnelSession(
         log.info { "Creating SSH client session." }
         val connectFuture: ConnectFuture =
             when (sshTunnel) {
-                SshNoTunnelMethod -> TODO("unreachable code")
+                SshNoTunnelMethod,
+                null -> TODO("unreachable code")
                 is SshKeyAuthTunnelMethod ->
                     client.connect(sshTunnel.user.trim(), sshTunnel.host.trim(), sshTunnel.port)
                 is SshPasswordAuthTunnelMethod ->

airbyte-cdk/bulk/core/base/src/main/resources/application.yaml

@@ -0,0 +1,4 @@
+airbyte:
+  file-transfer:
+    enabled: ${USE_FILE_TRANSFER:false}
+    staging-path: ${AIRBYTE_STAGING_DIRECTORY:/staging/files}

airbyte-cdk/bulk/core/base/src/test/kotlin/io/airbyte/cdk/initialization/TestApplicationYaml.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.cdk.initialization
+
+import io.micronaut.context.annotation.Bean
+import io.micronaut.context.annotation.Factory
+import io.micronaut.context.annotation.Value
+import io.micronaut.test.extensions.junit5.annotation.MicronautTest
+import jakarta.inject.Inject
+import kotlin.test.assertEquals
+import org.junit.jupiter.api.Test
+
+@MicronautTest
+class TestApplicationYaml {
+    @Inject lateinit var defaultValueBean: DefaultValueBean
+
+    @Test
+    fun testMainDefaultValue() {
+        assertEquals("/staging/files", defaultValueBean.stagingFolder)
+        assertEquals(false, defaultValueBean.fileTransferEnable)
+    }
+}
+
+data class DefaultValueBean(
+    val stagingFolder: String,
+    val fileTransferEnable: Boolean,
+)
+
+@Factory
+class TestFactory {
+    @Bean
+    fun defaultValueBean(
+        @Value("\${airbyte.file-transfer.staging-path}") stagingFolder: String,
+        @Value("\${airbyte.file-transfer.enabled}") fileTransferEnable: Boolean,
+    ): DefaultValueBean {
+        return DefaultValueBean(stagingFolder, fileTransferEnable)
+    }
+}

airbyte-cdk/bulk/core/extract/src/main/kotlin/io/airbyte/cdk/read/StateManager.kt

@@ -4,6 +4,7 @@ package io.airbyte.cdk.read
 import io.airbyte.cdk.StreamIdentifier
 import io.airbyte.cdk.asProtocolStreamDescriptor
 import io.airbyte.cdk.command.OpaqueStateValue
+import io.airbyte.cdk.util.Jsons
 import io.airbyte.protocol.models.v0.AirbyteGlobalState
 import io.airbyte.protocol.models.v0.AirbyteStateMessage
 import io.airbyte.protocol.models.v0.AirbyteStateStats
@@ -199,7 +200,9 @@ class StateManager(
                 streamStates.add(
                     AirbyteStreamState()
                         .withStreamDescriptor(streamID.asProtocolStreamDescriptor())
-                        .withStreamState(streamStateForCheckpoint.opaqueStateValue),
+                        .withStreamState(
+                            streamStateForCheckpoint.opaqueStateValue ?: Jsons.objectNode()
+                        ),
                 )
             }
             if (!shouldCheckpoint) {
@@ -233,7 +236,9 @@ class StateManager(
             val airbyteStreamState =
                 AirbyteStreamState()
                     .withStreamDescriptor(feed.id.asProtocolStreamDescriptor())
-                    .withStreamState(streamStateForCheckpoint.opaqueStateValue)
+                    .withStreamState(
+                        streamStateForCheckpoint.opaqueStateValue ?: Jsons.objectNode()
+                    )
             return AirbyteStateMessage()
                 .withType(AirbyteStateMessage.AirbyteStateType.STREAM)
                 .withStream(airbyteStreamState)

airbyte-cdk/bulk/core/load/build.gradle

@@ -25,7 +25,7 @@ dependencies {
     testFixturesImplementation "uk.org.webcompere:system-stubs-jupiter:2.1.7"
 }
 
-task integrationTest(type: Test) {
+def integrationTestTask = tasks.register('integrationTest', Test) {
     description = 'Runs the integration tests.'
     group = 'verification'
     testClassesDirs = sourceSets.integrationTest.output.classesDirs
@@ -37,9 +37,12 @@ task integrationTest(type: Test) {
     maxParallelForks = project.test.maxParallelForks
     maxHeapSize = project.test.maxHeapSize
 }
+// These tests are lightweight enough to run on every PR.
+tasks.named('check').configure {
+    dependsOn integrationTest
+}
+
 configurations {
     integrationTestImplementation.extendsFrom testImplementation
     integrationTestRuntimeOnly.extendsFrom testRuntimeOnly
 }
-// These tests are lightweight enough to run on every PR.
-rootProject.check.dependsOn(integrationTest)

airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt

@@ -8,6 +8,8 @@ import io.airbyte.cdk.load.test.util.NoopDestinationCleaner
 import io.airbyte.cdk.load.test.util.NoopExpectedRecordMapper
 import io.airbyte.cdk.load.test.util.NoopNameMapper
 import io.airbyte.cdk.load.write.BasicFunctionalityIntegrationTest
+import io.airbyte.cdk.load.write.Untyped
+import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Test
 
 class MockBasicFunctionalityIntegrationTest :
@@ -20,13 +22,19 @@ class MockBasicFunctionalityIntegrationTest :
         NoopNameMapper,
         isStreamSchemaRetroactive = false,
         supportsDedup = true,
+        stringifySchemalessObjects = false,
+        promoteUnionToObject = false,
+        preserveUndeclaredFields = true,
+        commitDataIncrementally = false,
+        allTypesBehavior = Untyped,
     ) {
     @Test
     override fun testBasicWrite() {
         super.testBasicWrite()
     }
 
     @Test
+    @Disabled
     override fun testMidSyncCheckpointingStreamState() {
         super.testMidSyncCheckpointingStreamState()
     }
@@ -46,6 +54,16 @@ class MockBasicFunctionalityIntegrationTest :
         super.testTruncateRefresh()
     }
 
+    @Test
+    override fun testInterruptedTruncateWithPriorData() {
+        super.testInterruptedTruncateWithPriorData()
+    }
+
+    @Test
+    override fun resumeAfterCancelledTruncate() {
+        super.resumeAfterCancelledTruncate()
+    }
+
     @Test
     override fun testAppend() {
         super.testAppend()
@@ -60,4 +78,19 @@ class MockBasicFunctionalityIntegrationTest :
     override fun testDedup() {
         super.testDedup()
     }
+
+    @Test
+    override fun testContainerTypes() {
+        super.testContainerTypes()
+    }
+
+    @Test
+    override fun testUnions() {
+        super.testUnions()
+    }
+
+    @Test
+    override fun testAllTypes() {
+        super.testAllTypes()
+    }
 }

airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationBackend.kt

@@ -78,6 +78,23 @@ object MockDestinationBackend {
         }
     }
 
+    fun commitFrom(srcFilename: String, dstFilename: String) {
+        val src = getFile(srcFilename)
+        insert(dstFilename, *src.toTypedArray())
+        src.clear()
+    }
+
+    fun commitAndDedupeFrom(
+        srcFilename: String,
+        dstFilename: String,
+        primaryKey: List<List<String>>,
+        cursor: List<String>,
+    ) {
+        val src = getFile(srcFilename)
+        upsert(dstFilename, primaryKey, cursor, *src.toTypedArray())
+        src.clear()
+    }
+
     fun readFile(filename: String): List<OutputRecord> {
         return getFile(filename)
     }