[Source-postgres]: Add logging in case of multiple records with same …

…LSN (#35842)
airbytehq · Mar 7, 2024 · 4fcff41 · 4fcff41
1 parent 8ab77d4
commit 4fcff41
Show file tree

Hide file tree

Showing 8 changed files with 16 additions and 9 deletions.
diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md
@@ -166,6 +166,7 @@ MavenLocal debugging steps:
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                        |
 |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.23.16 | 2024-03-06 | [\#35842](https://github.com/airbytehq/airbyte/pull/35842) | Improve logging in debezium processing.                                                                                                                        |
 | 0.23.15 | 2024-03-05 | [\#35827](https://github.com/airbytehq/airbyte/pull/35827) | improving the Junit interceptor.                                                                                                                               |
 | 0.23.14 | 2024-03-05 | [\#35739](https://github.com/airbytehq/airbyte/pull/35739) | Add logging to the CDC queue size. Fix the ContainerFactory.                                                                                                   |
 | 0.23.13 | 2024-03-04 | [\#35774](https://github.com/airbytehq/airbyte/pull/35774) | minor changes to the CDK test fixtures.                                                                                                                        |

diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties
@@ -1 +1 @@
-version=0.23.15
+version=0.23.16
diff --git a/.../java/io/airbyte/cdk/integrations/debezium/internals/DebeziumStateDecoratingIterator.java b/.../java/io/airbyte/cdk/integrations/debezium/internals/DebeziumStateDecoratingIterator.java
@@ -156,9 +156,12 @@ protected AirbyteMessage computeNext() {
 
         if (checkpointOffsetToSend.size() == 1
             && changeEventIterator.hasNext()
-            && !event.isSnapshotEvent()
-            && targetPosition.isEventAheadOffset(checkpointOffsetToSend, event)) {
-          sendCheckpointMessage = true;
+            && !event.isSnapshotEvent()) {
+          if (targetPosition.isEventAheadOffset(checkpointOffsetToSend, event)) {
+            sendCheckpointMessage = true;
+          } else {
+            LOGGER.info("Encountered {} records with the same event offset", recordsLastSync);
+          }
         }
       }
       recordsLastSync++;

diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle
@@ -12,7 +12,7 @@ java {
 }
 
 airbyteJavaConnector {
-    cdkVersionRequired = '0.20.6'
+    cdkVersionRequired = '0.23.16'
     features = ['db-sources', 'datastore-postgres']
     useLocalCdk = false
 }

diff --git a/airbyte-integrations/connectors/source-postgres/gradle.properties b/airbyte-integrations/connectors/source-postgres/gradle.properties
@@ -1 +1,3 @@
-testExecutionConcurrency=-1
+testExecutionConcurrency=-1
+
+JunitMethodExecutionTimeout=5 m
diff --git a/airbyte-integrations/connectors/source-postgres/metadata.yaml b/airbyte-integrations/connectors/source-postgres/metadata.yaml
@@ -9,7 +9,7 @@ data:
   connectorSubtype: database
   connectorType: source
   definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
-  dockerImageTag: 3.3.13
+  dockerImageTag: 3.3.14
   dockerRepository: airbyte/source-postgres
   documentationUrl: https://docs.airbyte.com/integrations/sources/postgres
   githubIssueLabel: source-postgres

diff --git a/...s/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresTestDatabase.java b/...s/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresTestDatabase.java
@@ -57,6 +57,7 @@ private ContainerModifier(String methodName) {
 
   }
 
+  @SuppressWarnings("deprecation")
   static public PostgresTestDatabase in(BaseImage baseImage, ContainerModifier... modifiers) {
     String[] methodNames = Stream.of(modifiers).map(im -> im.methodName).toList().toArray(new String[0]);
     final var container = new PostgresContainerFactory().shared(baseImage.reference, methodNames);

diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md
@@ -292,8 +292,8 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                    |
 |---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| 3.3.13  | 2024-02-27 | [35675](https://github.com/airbytehq/airbyte/pull/35675) | Fix invalid cdc error message.                                                                            |
-| 3.3.12  | 2024-02-22 | [35569](https://github.com/airbytehq/airbyte/pull/35569) | Fix logging bug. |
+| 3.3.14  | 2024-03-06 | [35842](https://github.com/airbytehq/airbyte/pull/35842) | Add logging to understand cases with a large number of records with the same LSN.                                                                                          |
+| 3.3.12  | 2024-02-22 | [35569](https://github.com/airbytehq/airbyte/pull/35569) | Fix logging bug.                                                                                                                                                           |
 | 3.3.11  | 2024-02-20 | [35304](https://github.com/airbytehq/airbyte/pull/35304) | Add config to throw an error on invalid CDC position and enable it by default.                                                                                             |
 | 3.3.10  | 2024-02-13 | [35036](https://github.com/airbytehq/airbyte/pull/34751) | Emit analytics message for invalid CDC cursor.                                                                                                                             |
 | 3.3.9   | 2024-02-13 | [35224](https://github.com/airbytehq/airbyte/pull/35224) | Adopt CDK 0.20.4                                                                                                                                                           |