[Source-Postgres] Update PostgresConverter to correctly stringify Big…

…Decimals without scientific notation (#31309)
airbytehq · Oct 13, 2023 · 896fa4b · 896fa4b
1 parent 1a76a8b
commit 896fa4b
Show file tree

Hide file tree

Showing 11 changed files with 21 additions and 9 deletions.
diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md
@@ -156,7 +156,8 @@ MavenLocal debugging steps:
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                        |
 |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| 0.1.8   | 2023-10-11 | [\#31322](https://github.com/airbytehq/airbyte/pull/31322) | Cap log line length to 32KB to prevent loss of records                                                                                                          |
+| 0.1.9   | 2023-10-12 | [\#31309](https://github.com/airbytehq/airbyte/pull/31309) | Use toPlainString() when handling BigDecimals in PostgresConverter                                                                                             |
+| 0.1.8   | 2023-10-11 | [\#31322](https://github.com/airbytehq/airbyte/pull/31322) | Cap log line length to 32KB to prevent loss of records                                                                                                         |
 | 0.1.7   | 2023-10-10 | [\#31194](https://github.com/airbytehq/airbyte/pull/31194) | Deallocate unused per stream buffer memory when empty                                                                                                          |
 | 0.1.6   | 2023-10-10 | [\#31083](https://github.com/airbytehq/airbyte/pull/31083) | Fix precision of numeric values in async destinations                                                                                                          |
 | 0.1.5   | 2023-10-09 | [\#31196](https://github.com/airbytehq/airbyte/pull/31196) | Update typo in CDK (CDN_LSN -> CDC_LSN)                                                                                                                        |

diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties
@@ -1 +1 @@
-version=0.1.8
+version=0.1.9
diff --git a/.../main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresConverter.java b/.../main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresConverter.java
@@ -125,7 +125,7 @@ private void registerNumber(final RelationalColumn field, final ConverterRegistr
       // The code below strips trailing zeros for integer numbers and represents number with exponent
       // if this number has decimals point.
       final double doubleValue = Double.parseDouble(x.toString());
-      var valueWithTruncatedZero = BigDecimal.valueOf(doubleValue).stripTrailingZeros().toString();
+      final String valueWithTruncatedZero = BigDecimal.valueOf(doubleValue).stripTrailingZeros().toPlainString();
       return valueWithTruncatedZero.contains(".") ? String.valueOf(doubleValue) : valueWithTruncatedZero;
     });
   }

diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle
@@ -4,7 +4,7 @@ plugins {
 }
 
 airbyteJavaConnector {
-    cdkVersionRequired = '0.1.8'
+    cdkVersionRequired = '0.1.9'
     features = ['db-sources']
     useLocalCdk = false
 }

diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-postgres-strict-encrypt/metadata.yaml
@@ -12,7 +12,7 @@ data:
   connectorType: source
   definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   maxSecondsBetweenMessages: 7200
-  dockerImageTag: 3.1.12
+  dockerImageTag: 3.1.13
   dockerRepository: airbyte/source-postgres-strict-encrypt
   githubIssueLabel: source-postgres
   icon: postgresql.svg

diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile
@@ -24,5 +24,5 @@ ENV APPLICATION source-postgres
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=3.1.12
+LABEL io.airbyte.version=3.1.13
 LABEL io.airbyte.name=airbyte/source-postgres
diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle
@@ -7,7 +7,7 @@ plugins {
 }
 
 airbyteJavaConnector {
-    cdkVersionRequired = '0.1.8'
+    cdkVersionRequired = '0.1.9'
     features = ['db-sources']
     useLocalCdk = false
 }

diff --git a/airbyte-integrations/connectors/source-postgres/metadata.yaml b/airbyte-integrations/connectors/source-postgres/metadata.yaml
@@ -6,7 +6,7 @@ data:
   connectorSubtype: database
   connectorType: source
   definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
-  dockerImageTag: 3.1.12
+  dockerImageTag: 3.1.13
   maxSecondsBetweenMessages: 7200
   dockerRepository: airbyte/source-postgres
   githubIssueLabel: source-postgres

diff --git a/...integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java b/...integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java
@@ -301,6 +301,16 @@ protected void initTests() {
             .addExpectedValues("33.345")
             .build());
 
+    // Verify that large integers are not deserialized into scientific notation
+    addDataTypeTestData(
+        TestDataHolder.builder()
+            .sourceType("numeric")
+            .airbyteType(JsonSchemaType.INTEGER)
+            .fullSourceDataType("NUMERIC(38)")
+            .addInsertValues("'70000'", "'853245'", "'900000000'")
+            .addExpectedValues("70000", "853245", "900000000")
+            .build());
+
     // case of a column type being a NUMERIC data type
     // with precision but no decimal
     addDataTypeTestData(

diff --git a/.../source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdkImportTest.java b/.../source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdkImportTest.java
@@ -21,7 +21,7 @@ class CdkImportTest {
   @Test
   void cdkVersionShouldMatch() {
     // Should fail in unit test phase:
-    assertEquals("0.1.8", CDKConstants.VERSION.replace("-SNAPSHOT", ""));
+    assertEquals("0.1.9", CDKConstants.VERSION.replace("-SNAPSHOT", ""));
   }
 
 }
diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md
@@ -291,6 +291,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                    |
 |---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 3.1.13  | 2023-10-13 | [31309](https://github.com/airbytehq/airbyte/pull/31309) | Addressed decimals being incorrectly deserialized into scientific notation.                                                                                                |
 | 3.1.12  | 2023-10-12 | [31328](https://github.com/airbytehq/airbyte/pull/31328) | Improvements to initial load of tables in older versions of postgres.                                                                                                      |
 | 3.1.11  | 2023-10-11 | [31322](https://github.com/airbytehq/airbyte/pull/31322) | Correct pevious release                                                                                                                                                    |
 | 3.1.10  | 2023-09-29 | [30806](https://github.com/airbytehq/airbyte/pull/30806) | Cap log line length to 32KB to prevent loss of records.                                                                                                                    |