feat: update destination dev-null for progressive rollout testing (#4…

…8693) Co-authored-by: Johnny Schmidt <[email protected]>
airbytehq · Nov 26, 2024 · d93f6f7 · d93f6f7
1 parent f4b3283
commit d93f6f7
Show file tree

Hide file tree

Showing 5 changed files with 64 additions and 37 deletions.
diff --git a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: file
   connectorType: destination
   definitionId: a7bcc9d8-13b3-4e49-b80d-d020b90045e3
-  dockerImageTag: 0.7.11
+  dockerImageTag: 0.7.12-rc.1
   dockerRepository: airbyte/destination-dev-null
   githubIssueLabel: destination-dev-null
   icon: airbyte.svg
@@ -14,6 +14,9 @@ data:
     oss:
       enabled: true
   releaseStage: alpha
+  releases:
+    rolloutConfiguration:
+      enableProgressiveRollout: true
   documentationUrl: https://docs.airbyte.com/integrations/destinations/dev-null
   tags:
     - language:java

diff --git a/...null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullConfiguration.kt b/...null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullConfiguration.kt
@@ -6,7 +6,9 @@ package io.airbyte.integrations.destination.dev_null
 
 import io.airbyte.cdk.load.command.DestinationConfiguration
 import io.airbyte.cdk.load.command.DestinationConfigurationFactory
+import io.github.oshai.kotlinlogging.KotlinLogging
 import io.micronaut.context.annotation.Factory
+import io.micronaut.context.annotation.Value
 import jakarta.inject.Singleton
 
 /** This is the simplified configuration object actually used by the implementation. */
@@ -27,6 +29,7 @@ data class Throttled(val millisPerRecord: Long) : DevNullType
 
 data class DevNullConfiguration(
     val type: DevNullType,
+    override val recordBatchSizeBytes: Long = 200 * 1024 * 1024,
 ) : DestinationConfiguration()
 
 /**
@@ -37,10 +40,13 @@ data class DevNullConfiguration(
  * rest of the implementation.
  */
 @Singleton
-class DevNullConfigurationFactory :
-    DestinationConfigurationFactory<DevNullSpecification, DevNullConfiguration> {
+class DevNullConfigurationFactory(
+    @Value("\${airbyte.destination.record-batch-size}") private val recordBatchSizeBytes: Long
+) : DestinationConfigurationFactory<DevNullSpecification, DevNullConfiguration> {
+    private val log = KotlinLogging.logger {}
 
     override fun makeWithoutExceptionHandling(pojo: DevNullSpecification): DevNullConfiguration {
+        log.info { "Record batch size from environment: $recordBatchSizeBytes" }
         return when (pojo) {
             is DevNullSpecificationOss -> {
                 when (pojo.testDestination) {
@@ -53,7 +59,7 @@ class DevNullConfigurationFactory :
                                             maxEntryCount =
                                                 pojo.testDestination.loggingConfig.maxEntryCount
                                                     .toInt(),
-                                        )
+                                        ),
                                 )
                             }
                             is EveryNthEntryConfig -> {
@@ -101,7 +107,7 @@ class DevNullConfigurationFactory :
                     }
                 }
             }
-        }
+        }.copy(recordBatchSizeBytes = recordBatchSizeBytes)
     }
 }
 

diff --git a/airbyte-integrations/connectors/destination-dev-null/src/main/resources/application.yaml b/airbyte-integrations/connectors/destination-dev-null/src/main/resources/application.yaml
@@ -8,3 +8,8 @@ airbyte:
   resources:
     disk:
       bytes: ${CONNECTOR_STORAGE_LIMIT_BYTES:5368709120} # 5GB
+  flush:
+    rate-ms: 900000 # 15 minutes
+    window-ms: 900000 # 15 minutes
+  destination:
+    record-batch-size: ${AIRBYTE_DESTINATION_RECORD_BATCH_SIZE:209715200}
diff --git a/...egrations/connectors/destination-dev-null/src/test-integration/resources/application.yaml b/...egrations/connectors/destination-dev-null/src/test-integration/resources/application.yaml
@@ -1,2 +1,14 @@
 deployment:
   mode: oss
+airbyte:
+  file-transfer:
+    enabled: ${USE_FILE_TRANSFER:false}
+    staging-path: ${AIRBYTE_STAGING_DIRECTORY:/staging/files}
+  resources:
+    disk:
+      bytes: ${CONNECTOR_STORAGE_LIMIT_BYTES:5368709120} # 5GB
+  flush:
+    rate-ms: 900000 # 15 minutes
+    window-ms: 900000 # 15 minutes
+  destination:
+    record-batch-size: 1 # Microbatch for testing
diff --git a/docs/integrations/destinations/dev-null.md b/docs/integrations/destinations/dev-null.md
@@ -47,37 +47,38 @@ This mode throws an exception after receiving a configurable number of messages.
 
 The OSS and Cloud variants have the same version number starting from version `0.2.2`.
 
-| Version | Date       | Pull Request                                             | Subject                                                                                      |
-|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------|
-| 0.7.11  | 2024-11-18 | [48468](https://github.com/airbytehq/airbyte/pull/48468) | Implement File CDk                                                                           |
-| 0.7.10  | 2024-11-08 | [48429](https://github.com/airbytehq/airbyte/pull/48429) | Bugfix: correctly handle state ID field                                                      |
-| 0.7.9   | 2024-11-07 | [48417](https://github.com/airbytehq/airbyte/pull/48417) | Only pass through the state ID field, not all additional properties                          |
-| 0.7.8   | 2024-11-07 | [48416](https://github.com/airbytehq/airbyte/pull/48416) | Bugfix: global state correclty sends additional properties                                   |
-| 0.7.7   | 2024-10-17 | [46692](https://github.com/airbytehq/airbyte/pull/46692) | Internal code changes                                                                        |
-| 0.7.6   | 2024-10-08 | [46683](https://github.com/airbytehq/airbyte/pull/46683) | Bugfix: pick up checkpoint safety check fix                                                  |
-| 0.7.5   | 2024-10-08 | [46683](https://github.com/airbytehq/airbyte/pull/46683) | Bugfix: checkpoints in order, all checkpoints processed before shutdown                      |
-| 0.7.4   | 2024-10-08 | [46650](https://github.com/airbytehq/airbyte/pull/46650) | Internal code changes                                                                        |
-| 0.7.3   | 2024-10-01 | [46559](https://github.com/airbytehq/airbyte/pull/46559) | From load CDK: async improvements, stream incomplete, additionalProperties on state messages |
-| 0.7.2   | 2024-10-01 | [45929](https://github.com/airbytehq/airbyte/pull/45929) | Internal code changes                                                                        |
-| 0.7.1   | 2024-09-30 | [46276](https://github.com/airbytehq/airbyte/pull/46276) | Upgrade to latest bulk CDK                                                                   |
-| 0.7.0   | 2024-09-20 | [45704](https://github.com/airbytehq/airbyte/pull/45704) |                                                                                              |
-| 0.6.1   | 2024-09-20 | [45715](https://github.com/airbytehq/airbyte/pull/45715) | add destination to cloud registry                                                            |
-| 0.6.0   | 2024-09-18 | [45651](https://github.com/airbytehq/airbyte/pull/45651) | merge destination-e2e(OSS) and destination-dev-null(cloud)                                   |
-| 0.5.0   | 2024-09-18 | [45650](https://github.com/airbytehq/airbyte/pull/45650) | upgrade cdk                                                                                  |
-| 0.4.1   | 2024-09-18 | [45649](https://github.com/airbytehq/airbyte/pull/45649) | convert test code to kotlin                                                                  |
-| 0.4.0   | 2024-09-18 | [45648](https://github.com/airbytehq/airbyte/pull/45648) | convert production code to kotlin                                                            |
-| 0.3.6   | 2024-05-09 | [38097](https://github.com/airbytehq/airbyte/pull/38097) | Support dedup                                                                                |
-| 0.3.5   | 2024-04-29 | [37366](https://github.com/airbytehq/airbyte/pull/37366) | Support refreshes                                                                            |
-| 0.3.4   | 2024-04-16 | [37366](https://github.com/airbytehq/airbyte/pull/37366) | Fix NPE                                                                                      |
-| 0.3.3   | 2024-04-16 | [37366](https://github.com/airbytehq/airbyte/pull/37366) | Fix Log trace messages                                                                       |
-| 0.3.2   | 2024-02-14 | [36812](https://github.com/airbytehq/airbyte/pull/36812) | Log trace messages                                                                           |
-| 0.3.1   | 2024-02-14 | [35278](https://github.com/airbytehq/airbyte/pull/35278) | Adopt CDK 0.20.6                                                                             |
-| 0.3.0   | 2023-05-08 | [25776](https://github.com/airbytehq/airbyte/pull/25776) | Standardize spec and change property field to non-keyword                                    |
-| 0.2.4   | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors                                       |
-| 0.2.3   | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                 |
-| 0.2.2   | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry.                                                                       |
-| 0.2.1   | 2021-12-19 | [\#8824](https://github.com/airbytehq/airbyte/pull/8905) | Fix documentation URL.                                                                       |
-| 0.2.0   | 2021-12-16 | [\#8824](https://github.com/airbytehq/airbyte/pull/8824) | Add multiple logging modes.                                                                  |
-| 0.1.0   | 2021-05-25 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) | Create initial version.                                                                      |
+| Version     | Date       | Pull Request                                             | Subject                                                                                      |
+|:------------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------|
+| 0.7.12-rc.1 | 2024-11-25 | [48693](https://github.com/airbytehq/airbyte/pull/48693) | Update for testing progressive rollout                                                       |
+| 0.7.11      | 2024-11-18 | [48468](https://github.com/airbytehq/airbyte/pull/48468) | Implement File CDk                                                                           |
+| 0.7.10      | 2024-11-08 | [48429](https://github.com/airbytehq/airbyte/pull/48429) | Bugfix: correctly handle state ID field                                                      |
+| 0.7.9       | 2024-11-07 | [48417](https://github.com/airbytehq/airbyte/pull/48417) | Only pass through the state ID field, not all additional properties                          |
+| 0.7.8       | 2024-11-07 | [48416](https://github.com/airbytehq/airbyte/pull/48416) | Bugfix: global state correclty sends additional properties                                   |
+| 0.7.7       | 2024-10-17 | [46692](https://github.com/airbytehq/airbyte/pull/46692) | Internal code changes                                                                        |
+| 0.7.6       | 2024-10-08 | [46683](https://github.com/airbytehq/airbyte/pull/46683) | Bugfix: pick up checkpoint safety check fix                                                  |
+| 0.7.5       | 2024-10-08 | [46683](https://github.com/airbytehq/airbyte/pull/46683) | Bugfix: checkpoints in order, all checkpoints processed before shutdown                      |
+| 0.7.4       | 2024-10-08 | [46650](https://github.com/airbytehq/airbyte/pull/46650) | Internal code changes                                                                        |
+| 0.7.3       | 2024-10-01 | [46559](https://github.com/airbytehq/airbyte/pull/46559) | From load CDK: async improvements, stream incomplete, additionalProperties on state messages |
+| 0.7.2       | 2024-10-01 | [45929](https://github.com/airbytehq/airbyte/pull/45929) | Internal code changes                                                                        |
+| 0.7.1       | 2024-09-30 | [46276](https://github.com/airbytehq/airbyte/pull/46276) | Upgrade to latest bulk CDK                                                                   |
+| 0.7.0       | 2024-09-20 | [45704](https://github.com/airbytehq/airbyte/pull/45704) |                                                                                              |
+| 0.6.1       | 2024-09-20 | [45715](https://github.com/airbytehq/airbyte/pull/45715) | add destination to cloud registry                                                            |
+| 0.6.0       | 2024-09-18 | [45651](https://github.com/airbytehq/airbyte/pull/45651) | merge destination-e2e(OSS) and destination-dev-null(cloud)                                   |
+| 0.5.0       | 2024-09-18 | [45650](https://github.com/airbytehq/airbyte/pull/45650) | upgrade cdk                                                                                  |
+| 0.4.1       | 2024-09-18 | [45649](https://github.com/airbytehq/airbyte/pull/45649) | convert test code to kotlin                                                                  |
+| 0.4.0       | 2024-09-18 | [45648](https://github.com/airbytehq/airbyte/pull/45648) | convert production code to kotlin                                                            |
+| 0.3.6       | 2024-05-09 | [38097](https://github.com/airbytehq/airbyte/pull/38097) | Support dedup                                                                                |
+| 0.3.5       | 2024-04-29 | [37366](https://github.com/airbytehq/airbyte/pull/37366) | Support refreshes                                                                            |
+| 0.3.4       | 2024-04-16 | [37366](https://github.com/airbytehq/airbyte/pull/37366) | Fix NPE                                                                                      |
+| 0.3.3       | 2024-04-16 | [37366](https://github.com/airbytehq/airbyte/pull/37366) | Fix Log trace messages                                                                       |
+| 0.3.2       | 2024-02-14 | [36812](https://github.com/airbytehq/airbyte/pull/36812) | Log trace messages                                                                           |
+| 0.3.1       | 2024-02-14 | [35278](https://github.com/airbytehq/airbyte/pull/35278) | Adopt CDK 0.20.6                                                                             |
+| 0.3.0       | 2023-05-08 | [25776](https://github.com/airbytehq/airbyte/pull/25776) | Standardize spec and change property field to non-keyword                                    |
+| 0.2.4       | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors                                       |
+| 0.2.3       | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                 |
+| 0.2.2       | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry.                                                                       |
+| 0.2.1       | 2021-12-19 | [\#8824](https://github.com/airbytehq/airbyte/pull/8905) | Fix documentation URL.                                                                       |
+| 0.2.0       | 2021-12-16 | [\#8824](https://github.com/airbytehq/airbyte/pull/8824) | Add multiple logging modes.                                                                  |
+| 0.1.0       | 2021-05-25 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) | Create initial version.                                                                      |
 
 </details>