From d2dc7921e95f922e373a2f4fe85216d8eed89b66 Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Thu, 21 Dec 2023 17:04:32 -0800 Subject: [PATCH 1/8] use base64 to encode binary --- .../integrations/source/mssql/MssqlSourceOperations.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java index 7fb984c6d7d4..f12f8c0afd30 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java @@ -29,6 +29,7 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import microsoft.sql.DateTimeOffset; +import org.apache.commons.codec.binary.Base64; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -128,7 +129,7 @@ protected void putBinary(final ObjectNode node, final int index) throws SQLException { final byte[] bytes = resultSet.getBytes(index); - final String value = new String(bytes, Charset.defaultCharset()); + final String value = Base64.encodeBase64String(bytes); node.put(columnName, value); } From 4176fa7b5e38e25768381290fe83290391cbe187 Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Fri, 22 Dec 2023 15:29:21 -0800 Subject: [PATCH 2/8] fix test --- .../source/mssql/AbstractMssqlSourceDatatypeTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java index f229caaad68f..50c8676cd5fb 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java @@ -257,7 +257,7 @@ protected void initTests() { .sourceType("binary") .airbyteType(JsonSchemaType.STRING_BASE_64) .addInsertValues("CAST( 'A' AS BINARY(1))", "null") - .addExpectedValues("A", null) + .addExpectedValues("QQ==", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); From b2349354a2b405b1faa4c7881fd63c878ecfabb3 Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Tue, 26 Dec 2023 10:47:08 -0800 Subject: [PATCH 3/8] format fix --- .../airbyte/integrations/source/mssql/MssqlSourceOperations.java | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java index f12f8c0afd30..4c46601f7fcf 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java @@ -18,7 +18,6 @@ import com.microsoft.sqlserver.jdbc.SQLServerResultSetMetaData; import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; import io.airbyte.protocol.models.JsonSchemaType; -import java.nio.charset.Charset; import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.ResultSet; From 5d9c9899cda047485bfd19bdfecf2f3843f9698c Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Tue, 2 Jan 2024 09:26:05 -0800 Subject: [PATCH 4/8] bump version up --- airbyte-integrations/connectors/source-mssql/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mssql/metadata.yaml b/airbyte-integrations/connectors/source-mssql/metadata.yaml index 4044481c816a..1f42d9ddff87 100644 --- a/airbyte-integrations/connectors/source-mssql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mssql/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 - dockerImageTag: 3.4.0 + dockerImageTag: 3.4.1 dockerRepository: airbyte/source-mssql documentationUrl: https://docs.airbyte.com/integrations/sources/mssql githubIssueLabel: source-mssql From 7d556edb6582e251b380753f5fce38a268f993c7 Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Tue, 2 Jan 2024 10:06:35 -0800 Subject: [PATCH 5/8] fix varbinary test --- .../source/mssql/AbstractMssqlSourceDatatypeTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java index 50c8676cd5fb..32c42ebea52c 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java @@ -267,7 +267,7 @@ protected void initTests() { .fullSourceDataType("varbinary(3)") .airbyteType(JsonSchemaType.STRING_BASE_64) .addInsertValues("CAST( 'ABC' AS VARBINARY)", "null") - .addExpectedValues("ABC", null) + .addExpectedValues("QUJD", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); From baa0a38a82613ab6ffd9fe10c45587528c290e41 Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Tue, 2 Jan 2024 10:28:44 -0800 Subject: [PATCH 6/8] tell debezium to convert to base64 for binary --- .../io/airbyte/integrations/source/mssql/MssqlCdcHelper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java index c9c38928beb2..08c211abe3b4 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java @@ -157,6 +157,8 @@ static Properties getDebeziumProperties(final JdbcDatabase database, final Confi props.setProperty("provide.transaction.metadata", "false"); props.setProperty("converters", "mssql_converter"); + props.setProperty("binary.handling.mode", "base64"); + props.setProperty("mssql_converter.type", MssqlDebeziumConverter.class.getName()); // If new stream(s) are added after a previously successful sync, From 4cea23e6e75695fa9f0814be979b151dc209aab4 Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Tue, 2 Jan 2024 10:31:46 -0800 Subject: [PATCH 7/8] add doc --- docs/integrations/sources/mssql.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index 578484b2d0b2..1e1059f3150b 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -342,6 +342,7 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.4.1 | 2024-01-02 | [33755](https://github.com/airbytehq/airbyte/pull/33755) | Encode binary to base64 format | | 3.4.0 | 2023-12-19 | [33481](https://github.com/airbytehq/airbyte/pull/33481) | Remove LEGACY state flag | | 3.3.2 | 2023-12-14 | [33505](https://github.com/airbytehq/airbyte/pull/33225) | Using the released CDK. | | 3.3.1 | 2023-12-12 | [33225](https://github.com/airbytehq/airbyte/pull/33225) | extracting MsSql specific files out of the CDK. | From fcb4f87937f6184e85ff4d216ebb2ec2bd54e82d Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Wed, 3 Jan 2024 16:02:19 -0800 Subject: [PATCH 8/8] use custom converter instead --- .../io/airbyte/integrations/source/mssql/MssqlCdcHelper.java | 1 - .../integrations/source/mssql/MssqlDebeziumConverter.java | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java index 08c211abe3b4..c330ea5267bb 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java @@ -157,7 +157,6 @@ static Properties getDebeziumProperties(final JdbcDatabase database, final Confi props.setProperty("provide.transaction.metadata", "false"); props.setProperty("converters", "mssql_converter"); - props.setProperty("binary.handling.mode", "base64"); props.setProperty("mssql_converter.type", MssqlDebeziumConverter.class.getName()); diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlDebeziumConverter.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlDebeziumConverter.java index 0e57af398ec9..69ffcf2c0499 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlDebeziumConverter.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlDebeziumConverter.java @@ -13,7 +13,6 @@ import io.debezium.spi.converter.CustomConverter; import io.debezium.spi.converter.RelationalColumn; import java.math.BigDecimal; -import java.nio.charset.Charset; import java.sql.Timestamp; import java.time.LocalDateTime; import java.time.OffsetDateTime; @@ -22,6 +21,7 @@ import java.util.Properties; import java.util.Set; import microsoft.sql.DateTimeOffset; +import org.apache.commons.codec.binary.Base64; import org.apache.kafka.connect.data.SchemaBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -197,7 +197,7 @@ private void registerBinary(final RelationalColumn field, } if (input instanceof byte[]) { - return new String((byte[]) input, Charset.defaultCharset()); + return Base64.encodeBase64String((byte[]) input); } LOGGER.warn("Uncovered binary class type '{}'. Use default converter",