From 8d80b599cd58f539c3896dec65ae64198b96acbe Mon Sep 17 00:00:00 2001 From: duanzhengqiang Date: Sun, 22 Dec 2024 20:17:05 +0800 Subject: [PATCH] Fix mysql longblob wrong column type returned by proxy protocol --- .../mysql/constant/MySQLBinaryColumnType.java | 6 +- .../mysql/constant/MySQLCharacterSet.java | 1 + .../constant/MySQLBinaryColumnTypeTest.java | 6 +- .../query/builder/ResponsePacketBuilder.java | 14 +++- .../e2e/engine/type/dql/BaseDQLE2EIT.java | 7 +- .../dql/e2e-dql-select-with-data-types.xml | 33 ++++++++ .../data/actual/dataset/mysql/dataset.xml | 81 +++++++++++++++++++ .../actual/init-sql/mysql/01-actual-init.sql | 3 + .../data/expected/dataset/mysql/dataset.xml | 81 +++++++++++++++++++ .../init-sql/mysql/01-expected-init.sql | 3 + 10 files changed, 224 insertions(+), 11 deletions(-) create mode 100644 test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-with-data-types.xml create mode 100644 test/e2e/sql/src/test/resources/env/scenario/passthrough/data/actual/dataset/mysql/dataset.xml create mode 100644 test/e2e/sql/src/test/resources/env/scenario/passthrough/data/expected/dataset/mysql/dataset.xml diff --git a/db-protocol/mysql/src/main/java/org/apache/shardingsphere/db/protocol/mysql/constant/MySQLBinaryColumnType.java b/db-protocol/mysql/src/main/java/org/apache/shardingsphere/db/protocol/mysql/constant/MySQLBinaryColumnType.java index 069948f196155..5289927a84bea 100644 --- a/db-protocol/mysql/src/main/java/org/apache/shardingsphere/db/protocol/mysql/constant/MySQLBinaryColumnType.java +++ b/db-protocol/mysql/src/main/java/org/apache/shardingsphere/db/protocol/mysql/constant/MySQLBinaryColumnType.java @@ -125,9 +125,9 @@ public enum MySQLBinaryColumnType implements BinaryColumnType { JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.DATE, DATE); JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.TIME, TIME); JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.TIMESTAMP, TIMESTAMP); - JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.BINARY, STRING); - JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.VARBINARY, VAR_STRING); - JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.LONGVARBINARY, VAR_STRING); + JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.BINARY, LONG_BLOB); + JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.VARBINARY, TINY_BLOB); + JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.LONGVARBINARY, BLOB); JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.NULL, NULL); JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.BLOB, BLOB); for (MySQLBinaryColumnType each : values()) { diff --git a/db-protocol/mysql/src/main/java/org/apache/shardingsphere/db/protocol/mysql/constant/MySQLCharacterSet.java b/db-protocol/mysql/src/main/java/org/apache/shardingsphere/db/protocol/mysql/constant/MySQLCharacterSet.java index 749938c4e01b4..f96994fff8e48 100644 --- a/db-protocol/mysql/src/main/java/org/apache/shardingsphere/db/protocol/mysql/constant/MySQLCharacterSet.java +++ b/db-protocol/mysql/src/main/java/org/apache/shardingsphere/db/protocol/mysql/constant/MySQLCharacterSet.java @@ -101,6 +101,7 @@ public enum MySQLCharacterSet { UTF32_GENERAL_CI(60, () -> Charset.forName("utf32")), UTF32_BIN(61, () -> Charset.forName("utf32")), UTF16LE_BIN(62, () -> StandardCharsets.UTF_16LE), + BINARY(63, () -> Charset.forName("binary")), ARMSCII8_BIN(64, () -> Charset.forName("armscii8")), ASCII_BIN(65, () -> StandardCharsets.US_ASCII), CP1250_BIN(66, () -> Charset.forName("cp1250")), diff --git a/db-protocol/mysql/src/test/java/org/apache/shardingsphere/db/protocol/mysql/constant/MySQLBinaryColumnTypeTest.java b/db-protocol/mysql/src/test/java/org/apache/shardingsphere/db/protocol/mysql/constant/MySQLBinaryColumnTypeTest.java index 6e387c6135ee5..5670137cc6793 100644 --- a/db-protocol/mysql/src/test/java/org/apache/shardingsphere/db/protocol/mysql/constant/MySQLBinaryColumnTypeTest.java +++ b/db-protocol/mysql/src/test/java/org/apache/shardingsphere/db/protocol/mysql/constant/MySQLBinaryColumnTypeTest.java @@ -45,9 +45,9 @@ void assertValueOfJDBC() { assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.DATE), is(MySQLBinaryColumnType.DATE)); assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.TIME), is(MySQLBinaryColumnType.TIME)); assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.TIMESTAMP), is(MySQLBinaryColumnType.TIMESTAMP)); - assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.BINARY), is(MySQLBinaryColumnType.STRING)); - assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.VARBINARY), is(MySQLBinaryColumnType.VAR_STRING)); - assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.LONGVARBINARY), is(MySQLBinaryColumnType.VAR_STRING)); + assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.BINARY), is(MySQLBinaryColumnType.LONG_BLOB)); + assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.VARBINARY), is(MySQLBinaryColumnType.TINY_BLOB)); + assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.LONGVARBINARY), is(MySQLBinaryColumnType.BLOB)); assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.NULL), is(MySQLBinaryColumnType.NULL)); assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.BLOB), is(MySQLBinaryColumnType.BLOB)); } diff --git a/proxy/frontend/type/mysql/src/main/java/org/apache/shardingsphere/proxy/frontend/mysql/command/query/builder/ResponsePacketBuilder.java b/proxy/frontend/type/mysql/src/main/java/org/apache/shardingsphere/proxy/frontend/mysql/command/query/builder/ResponsePacketBuilder.java index d244835efb673..ba6b0dc8239d1 100644 --- a/proxy/frontend/type/mysql/src/main/java/org/apache/shardingsphere/proxy/frontend/mysql/command/query/builder/ResponsePacketBuilder.java +++ b/proxy/frontend/type/mysql/src/main/java/org/apache/shardingsphere/proxy/frontend/mysql/command/query/builder/ResponsePacketBuilder.java @@ -20,6 +20,7 @@ import lombok.AccessLevel; import lombok.NoArgsConstructor; import org.apache.shardingsphere.db.protocol.mysql.constant.MySQLBinaryColumnType; +import org.apache.shardingsphere.db.protocol.mysql.constant.MySQLCharacterSet; import org.apache.shardingsphere.db.protocol.mysql.packet.command.query.MySQLColumnDefinition41Packet; import org.apache.shardingsphere.db.protocol.mysql.packet.command.query.MySQLColumnDefinitionFlag; import org.apache.shardingsphere.db.protocol.mysql.packet.command.query.MySQLFieldCountPacket; @@ -30,8 +31,11 @@ import org.apache.shardingsphere.proxy.backend.response.header.query.QueryResponseHeader; import org.apache.shardingsphere.proxy.backend.response.header.update.UpdateResponseHeader; +import java.sql.Types; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -45,19 +49,22 @@ public final class ResponsePacketBuilder { private static final String BLOB_COLUMN_TYPE_KEYWORD = "BLOB"; + private static final Collection BINARY_TYPES = new HashSet<>(Arrays.asList(Types.BINARY, Types.VARBINARY, Types.LONGVARBINARY)); + /** * Build query response packets. * * @param queryResponseHeader query response header - * @param characterSet MySQL character set id + * @param sessionCharacterSet MySQL character set id * @param statusFlags server status flags * @return query response packets */ - public static Collection buildQueryResponsePackets(final QueryResponseHeader queryResponseHeader, final int characterSet, final int statusFlags) { + public static Collection buildQueryResponsePackets(final QueryResponseHeader queryResponseHeader, final int sessionCharacterSet, final int statusFlags) { Collection result = new LinkedList<>(); List queryHeaders = queryResponseHeader.getQueryHeaders(); result.add(new MySQLFieldCountPacket(queryHeaders.size())); for (QueryHeader each : queryHeaders) { + int characterSet = BINARY_TYPES.contains(each.getColumnType()) ? MySQLCharacterSet.BINARY.getId() : sessionCharacterSet; result.add(new MySQLColumnDefinition41Packet(characterSet, getColumnDefinitionFlag(each), each.getSchema(), each.getTable(), each.getTable(), each.getColumnLabel(), each.getColumnName(), each.getColumnLength(), MySQLBinaryColumnType.valueOfJDBCType(each.getColumnType()), each.getDecimals(), false)); } @@ -82,6 +89,9 @@ private static int getColumnDefinitionFlag(final QueryHeader header) { if (header.getColumnTypeName().contains(BINARY_COLUMN_TYPE_KEYWORD) || header.getColumnTypeName().contains(BLOB_COLUMN_TYPE_KEYWORD)) { result += MySQLColumnDefinitionFlag.BINARY_COLLATION.getValue(); } + if (BINARY_TYPES.contains(header.getColumnType())) { + result += MySQLColumnDefinitionFlag.BLOB.getValue(); + } return result; } diff --git a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/engine/type/dql/BaseDQLE2EIT.java b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/engine/type/dql/BaseDQLE2EIT.java index 63bf12a0f084c..5d832572b2cbe 100644 --- a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/engine/type/dql/BaseDQLE2EIT.java +++ b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/engine/type/dql/BaseDQLE2EIT.java @@ -23,10 +23,10 @@ import org.apache.shardingsphere.test.e2e.cases.dataset.metadata.DataSetColumn; import org.apache.shardingsphere.test.e2e.cases.dataset.metadata.DataSetMetaData; import org.apache.shardingsphere.test.e2e.cases.dataset.row.DataSetRow; -import org.apache.shardingsphere.test.e2e.env.E2EEnvironmentAware; -import org.apache.shardingsphere.test.e2e.env.E2EEnvironmentEngine; import org.apache.shardingsphere.test.e2e.engine.context.E2ETestContext; import org.apache.shardingsphere.test.e2e.env.DataSetEnvironmentManager; +import org.apache.shardingsphere.test.e2e.env.E2EEnvironmentAware; +import org.apache.shardingsphere.test.e2e.env.E2EEnvironmentEngine; import org.apache.shardingsphere.test.e2e.env.runtime.scenario.path.ScenarioDataPath; import org.apache.shardingsphere.test.e2e.env.runtime.scenario.path.ScenarioDataPath.Type; import org.apache.shardingsphere.test.e2e.framework.param.model.AssertionTestParameter; @@ -136,7 +136,8 @@ private void assertMetaData(final ResultSetMetaData actualResultSetMetaData, fin if ("db_tbl_sql_federation".equals(testParam.getScenario())) { continue; } - if ("jdbc".equals(testParam.getAdapter()) && "Cluster".equals(testParam.getMode()) && "encrypt".equals(testParam.getScenario())) { + if ("jdbc".equals(testParam.getAdapter()) && "Cluster".equals(testParam.getMode()) && "encrypt".equals(testParam.getScenario()) + || "passthrough".equals(testParam.getScenario())) { // FIXME correct columnType with proxy adapter and other jdbc scenario assertThat(actualResultSetMetaData.getColumnType(i + 1), is(expectedResultSetMetaData.getColumnType(i + 1))); } diff --git a/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-with-data-types.xml b/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-with-data-types.xml new file mode 100644 index 0000000000000..431973ebaff40 --- /dev/null +++ b/test/e2e/sql/src/test/resources/cases/dql/e2e-dql-select-with-data-types.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + diff --git a/test/e2e/sql/src/test/resources/env/scenario/passthrough/data/actual/dataset/mysql/dataset.xml b/test/e2e/sql/src/test/resources/env/scenario/passthrough/data/actual/dataset/mysql/dataset.xml new file mode 100644 index 0000000000000..0353531c0a8a1 --- /dev/null +++ b/test/e2e/sql/src/test/resources/env/scenario/passthrough/data/actual/dataset/mysql/dataset.xml @@ -0,0 +1,81 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/e2e/sql/src/test/resources/env/scenario/passthrough/data/actual/init-sql/mysql/01-actual-init.sql b/test/e2e/sql/src/test/resources/env/scenario/passthrough/data/actual/init-sql/mysql/01-actual-init.sql index 8d3c9dad53b0d..350af11696813 100644 --- a/test/e2e/sql/src/test/resources/env/scenario/passthrough/data/actual/init-sql/mysql/01-actual-init.sql +++ b/test/e2e/sql/src/test/resources/env/scenario/passthrough/data/actual/init-sql/mysql/01-actual-init.sql @@ -29,3 +29,6 @@ CREATE TABLE passthrough.t_data_type_money (id INT PRIMARY KEY, val NUMERIC(16, CREATE TABLE passthrough.t_data_type_bytea (id INT PRIMARY KEY, val BLOB NOT NULL); CREATE TABLE passthrough.t_data_type_date (id INT PRIMARY KEY, creation_date DATE NOT NULL, update_date DATETIME NOT NULL); CREATE TABLE passthrough.t_data_type_uuid (id INT PRIMARY KEY, val VARCHAR(36) NOT NULL); +CREATE TABLE passthrough.t_data_type_binary (id INT PRIMARY KEY, val BINARY(10) NOT NULL); +CREATE TABLE passthrough.t_data_type_varbinary (id INT PRIMARY KEY, val VARBINARY(10) NOT NULL); +CREATE TABLE passthrough.t_data_type_longblob (id INT PRIMARY KEY, val LONGBLOB NOT NULL); diff --git a/test/e2e/sql/src/test/resources/env/scenario/passthrough/data/expected/dataset/mysql/dataset.xml b/test/e2e/sql/src/test/resources/env/scenario/passthrough/data/expected/dataset/mysql/dataset.xml new file mode 100644 index 0000000000000..358c0b93bb4df --- /dev/null +++ b/test/e2e/sql/src/test/resources/env/scenario/passthrough/data/expected/dataset/mysql/dataset.xml @@ -0,0 +1,81 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/e2e/sql/src/test/resources/env/scenario/passthrough/data/expected/init-sql/mysql/01-expected-init.sql b/test/e2e/sql/src/test/resources/env/scenario/passthrough/data/expected/init-sql/mysql/01-expected-init.sql index 2e2c97ea4a871..3c5fc3aa0f307 100644 --- a/test/e2e/sql/src/test/resources/env/scenario/passthrough/data/expected/init-sql/mysql/01-expected-init.sql +++ b/test/e2e/sql/src/test/resources/env/scenario/passthrough/data/expected/init-sql/mysql/01-expected-init.sql @@ -29,3 +29,6 @@ CREATE TABLE expected_dataset.t_data_type_money (id INT PRIMARY KEY, val NUMERI CREATE TABLE expected_dataset.t_data_type_bytea (id INT PRIMARY KEY, val BLOB NOT NULL); CREATE TABLE expected_dataset.t_data_type_date (id INT PRIMARY KEY, creation_date DATE NOT NULL, update_date DATETIME NOT NULL); CREATE TABLE expected_dataset.t_data_type_uuid (id INT PRIMARY KEY, val VARCHAR(36) NOT NULL); +CREATE TABLE expected_dataset.t_data_type_binary (id INT PRIMARY KEY, val BINARY(10) NOT NULL); +CREATE TABLE expected_dataset.t_data_type_varbinary (id INT PRIMARY KEY, val VARBINARY(10) NOT NULL); +CREATE TABLE expected_dataset.t_data_type_longblob (id INT PRIMARY KEY, val LONGBLOB NOT NULL);