Skip to content

Commit

Permalink
Fix mysql longblob wrong column type returned by proxy protocol (#34121)
Browse files Browse the repository at this point in the history
* Fix mysql longblob wrong column type returned by proxy protocol

* update release note

* remove sphereex comment

* only open mysql passthrough columnType assert
  • Loading branch information
strongduanmu authored Dec 23, 2024
1 parent 646d4d8 commit 1f3af26
Show file tree
Hide file tree
Showing 11 changed files with 223 additions and 11 deletions.
1 change: 1 addition & 0 deletions RELEASE-NOTES.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
1. SQL Binder: Fixes the expression segment cannot find the outer table when binding - [#34015](https://github.com/apache/shardingsphere/pull/34015)
1. Proxy: Fixes "ALL PRIVILEGES ON `DB`.*" is not recognized during SELECT privilege verification for MySQL - [#34037](https://github.com/apache/shardingsphere/pull/34037)
1. Encrypt: Use sql bind info in EncryptInsertPredicateColumnTokenGenerator to avoid wrong column table mapping - [#34110](https://github.com/apache/shardingsphere/pull/34110)
1. Proxy: Fix mysql longblob wrong column type returned by proxy protocol - [#34121](https://github.com/apache/shardingsphere/pull/34121)

### Change Logs

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,9 @@ public enum MySQLBinaryColumnType implements BinaryColumnType {
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.DATE, DATE);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.TIME, TIME);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.TIMESTAMP, TIMESTAMP);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.BINARY, STRING);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.VARBINARY, VAR_STRING);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.LONGVARBINARY, VAR_STRING);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.BINARY, LONG_BLOB);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.VARBINARY, TINY_BLOB);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.LONGVARBINARY, BLOB);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.NULL, NULL);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.BLOB, BLOB);
for (MySQLBinaryColumnType each : values()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ public enum MySQLCharacterSet {
UTF32_GENERAL_CI(60, () -> Charset.forName("utf32")),
UTF32_BIN(61, () -> Charset.forName("utf32")),
UTF16LE_BIN(62, () -> StandardCharsets.UTF_16LE),
BINARY(63, () -> Charset.forName("binary")),
ARMSCII8_BIN(64, () -> Charset.forName("armscii8")),
ASCII_BIN(65, () -> StandardCharsets.US_ASCII),
CP1250_BIN(66, () -> Charset.forName("cp1250")),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ void assertValueOfJDBC() {
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.DATE), is(MySQLBinaryColumnType.DATE));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.TIME), is(MySQLBinaryColumnType.TIME));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.TIMESTAMP), is(MySQLBinaryColumnType.TIMESTAMP));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.BINARY), is(MySQLBinaryColumnType.STRING));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.VARBINARY), is(MySQLBinaryColumnType.VAR_STRING));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.LONGVARBINARY), is(MySQLBinaryColumnType.VAR_STRING));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.BINARY), is(MySQLBinaryColumnType.LONG_BLOB));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.VARBINARY), is(MySQLBinaryColumnType.TINY_BLOB));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.LONGVARBINARY), is(MySQLBinaryColumnType.BLOB));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.NULL), is(MySQLBinaryColumnType.NULL));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.BLOB), is(MySQLBinaryColumnType.BLOB));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import org.apache.shardingsphere.db.protocol.mysql.constant.MySQLBinaryColumnType;
import org.apache.shardingsphere.db.protocol.mysql.constant.MySQLCharacterSet;
import org.apache.shardingsphere.db.protocol.mysql.packet.command.query.MySQLColumnDefinition41Packet;
import org.apache.shardingsphere.db.protocol.mysql.packet.command.query.MySQLColumnDefinitionFlag;
import org.apache.shardingsphere.db.protocol.mysql.packet.command.query.MySQLFieldCountPacket;
Expand All @@ -30,8 +31,11 @@
import org.apache.shardingsphere.proxy.backend.response.header.query.QueryResponseHeader;
import org.apache.shardingsphere.proxy.backend.response.header.update.UpdateResponseHeader;

import java.sql.Types;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;

Expand All @@ -45,19 +49,22 @@ public final class ResponsePacketBuilder {

private static final String BLOB_COLUMN_TYPE_KEYWORD = "BLOB";

private static final Collection<Integer> BINARY_TYPES = new HashSet<>(Arrays.asList(Types.BINARY, Types.VARBINARY, Types.LONGVARBINARY));

/**
* Build query response packets.
*
* @param queryResponseHeader query response header
* @param characterSet MySQL character set id
* @param sessionCharacterSet MySQL character set id
* @param statusFlags server status flags
* @return query response packets
*/
public static Collection<DatabasePacket> buildQueryResponsePackets(final QueryResponseHeader queryResponseHeader, final int characterSet, final int statusFlags) {
public static Collection<DatabasePacket> buildQueryResponsePackets(final QueryResponseHeader queryResponseHeader, final int sessionCharacterSet, final int statusFlags) {
Collection<DatabasePacket> result = new LinkedList<>();
List<QueryHeader> queryHeaders = queryResponseHeader.getQueryHeaders();
result.add(new MySQLFieldCountPacket(queryHeaders.size()));
for (QueryHeader each : queryHeaders) {
int characterSet = BINARY_TYPES.contains(each.getColumnType()) ? MySQLCharacterSet.BINARY.getId() : sessionCharacterSet;
result.add(new MySQLColumnDefinition41Packet(characterSet, getColumnDefinitionFlag(each), each.getSchema(), each.getTable(), each.getTable(),
each.getColumnLabel(), each.getColumnName(), each.getColumnLength(), MySQLBinaryColumnType.valueOfJDBCType(each.getColumnType()), each.getDecimals(), false));
}
Expand All @@ -82,6 +89,9 @@ private static int getColumnDefinitionFlag(final QueryHeader header) {
if (header.getColumnTypeName().contains(BINARY_COLUMN_TYPE_KEYWORD) || header.getColumnTypeName().contains(BLOB_COLUMN_TYPE_KEYWORD)) {
result += MySQLColumnDefinitionFlag.BINARY_COLLATION.getValue();
}
if (BINARY_TYPES.contains(header.getColumnType())) {
result += MySQLColumnDefinitionFlag.BLOB.getValue();
}
return result;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@
import org.apache.shardingsphere.test.e2e.cases.dataset.metadata.DataSetColumn;
import org.apache.shardingsphere.test.e2e.cases.dataset.metadata.DataSetMetaData;
import org.apache.shardingsphere.test.e2e.cases.dataset.row.DataSetRow;
import org.apache.shardingsphere.test.e2e.env.E2EEnvironmentAware;
import org.apache.shardingsphere.test.e2e.env.E2EEnvironmentEngine;
import org.apache.shardingsphere.test.e2e.engine.context.E2ETestContext;
import org.apache.shardingsphere.test.e2e.env.DataSetEnvironmentManager;
import org.apache.shardingsphere.test.e2e.env.E2EEnvironmentAware;
import org.apache.shardingsphere.test.e2e.env.E2EEnvironmentEngine;
import org.apache.shardingsphere.test.e2e.env.runtime.scenario.path.ScenarioDataPath;
import org.apache.shardingsphere.test.e2e.env.runtime.scenario.path.ScenarioDataPath.Type;
import org.apache.shardingsphere.test.e2e.framework.param.model.AssertionTestParameter;
Expand Down Expand Up @@ -136,7 +136,8 @@ private void assertMetaData(final ResultSetMetaData actualResultSetMetaData, fin
if ("db_tbl_sql_federation".equals(testParam.getScenario())) {
continue;
}
if ("jdbc".equals(testParam.getAdapter()) && "Cluster".equals(testParam.getMode()) && "encrypt".equals(testParam.getScenario())) {
if ("jdbc".equals(testParam.getAdapter()) && "Cluster".equals(testParam.getMode()) && "encrypt".equals(testParam.getScenario())
|| "MySQL".equals(testParam.getDatabaseType().getType()) && "passthrough".equals(testParam.getScenario())) {
// FIXME correct columnType with proxy adapter and other jdbc scenario
assertThat(actualResultSetMetaData.getColumnType(i + 1), is(expectedResultSetMetaData.getColumnType(i + 1)));
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<e2e-test-cases>
<!--<test-case sql="SELECT * FROM t_data_type_binary WHERE id = ?" db-types="MySQL" scenario-types="passthrough">
<assertion parameters="1:int" expected-data-source-name="expected_dataset" />
</test-case>-->

<test-case sql="SELECT * FROM t_data_type_varbinary WHERE id = ?" db-types="MySQL" scenario-types="passthrough">
<assertion parameters="1:int" expected-data-source-name="expected_dataset" />
</test-case>

<test-case sql="SELECT * FROM t_data_type_longblob WHERE id = ?" db-types="MySQL" scenario-types="passthrough">
<assertion parameters="1:int" expected-data-source-name="expected_dataset" />
</test-case>
</e2e-test-cases>
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<dataset>
<metadata data-nodes="passthrough.t_data_type_integer">
<column name="id" type="numeric" />
<column name="col_bigint" type="numeric" />
<column name="col_int" type="numeric" />
<column name="col_mediumint" type="numeric" />
<column name="col_smallint" type="numeric" />
<column name="col_tinyint" type="numeric" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_integer_unsigned">
<column name="id" type="numeric" />
<column name="col_bigint_unsigned" type="decimal" />
<column name="col_int_unsigned" type="numeric" />
<column name="col_mediumint_unsigned" type="numeric" />
<column name="col_smallint_unsigned" type="numeric" />
<column name="col_tinyint_unsigned" type="numeric" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_floating_point">
<column name="id" type="numeric" />
<column name="col_float" type="numeric" />
<column name="col_double" type="numeric" />
</metadata>
<metadata data-nodes="passthrough.t_with_generated_id">
<column name="id" type="numeric" />
<column name="val" type="varchar" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_money">
<column name="id" type="numeric" />
<column name="val" type="cast#money" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_bytea">
<column name="id" type="numeric" />
<column name="val" type="varchar" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_uuid">
<column name="id" type="numeric" />
<column name="val" type="varchar" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_date">
<column name="id" type="numeric" />
<column name="creation_date" type="Date" />
<column name="update_date" type="datetime" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_binary">
<column name="id" type="numeric" />
<column name="val" type="binary" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_varbinary">
<column name="id" type="numeric" />
<column name="val" type="varbinary" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_longblob">
<column name="id" type="numeric" />
<column name="val" type="longblob" />
</metadata>
<row data-node="passthrough.t_data_type_integer_unsigned" values="1001, 18446744073709551615, 4294967295, 16777215, 65535, 255" />
<row data-node="passthrough.t_data_type_integer_unsigned" values="1002, 0, 0, 0, 0, 0" />
<row data-node="passthrough.t_data_type_money" values="1001, 123" />
<row data-node="passthrough.t_data_type_money" values="1002, 456" />
<row data-node="passthrough.t_data_type_date" values="1, 2017-08-08, 2017-08-08 00:00:00" />
<row data-node="passthrough.t_data_type_binary" values="1, '123'" />
<row data-node="passthrough.t_data_type_varbinary" values="1, '123'" />
<row data-node="passthrough.t_data_type_longblob" values="1, '123'" />
</dataset>
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,6 @@ CREATE TABLE passthrough.t_data_type_money (id INT PRIMARY KEY, val NUMERIC(16,
CREATE TABLE passthrough.t_data_type_bytea (id INT PRIMARY KEY, val BLOB NOT NULL);
CREATE TABLE passthrough.t_data_type_date (id INT PRIMARY KEY, creation_date DATE NOT NULL, update_date DATETIME NOT NULL);
CREATE TABLE passthrough.t_data_type_uuid (id INT PRIMARY KEY, val VARCHAR(36) NOT NULL);
CREATE TABLE passthrough.t_data_type_binary (id INT PRIMARY KEY, val BINARY(10) NOT NULL);
CREATE TABLE passthrough.t_data_type_varbinary (id INT PRIMARY KEY, val VARBINARY(10) NOT NULL);
CREATE TABLE passthrough.t_data_type_longblob (id INT PRIMARY KEY, val LONGBLOB NOT NULL);
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<dataset>
<metadata data-nodes="expected_dataset.t_data_type_integer">
<column name="id" type="numeric" />
<column name="col_bigint" type="numeric" />
<column name="col_int" type="numeric" />
<column name="col_mediumint" type="numeric" />
<column name="col_smallint" type="numeric" />
<column name="col_tinyint" type="numeric" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_integer_unsigned">
<column name="id" type="numeric" />
<column name="col_bigint_unsigned" type="decimal" />
<column name="col_int_unsigned" type="numeric" />
<column name="col_mediumint_unsigned" type="numeric" />
<column name="col_smallint_unsigned" type="numeric" />
<column name="col_tinyint_unsigned" type="numeric" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_floating_point">
<column name="id" type="numeric" />
<column name="col_float" type="numeric" />
<column name="col_double" type="numeric" />
</metadata>
<metadata data-nodes="expected_dataset.t_with_generated_id">
<column name="id" type="numeric" />
<column name="val" type="varchar" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_money">
<column name="id" type="numeric" />
<column name="val" type="cast#money" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_bytea">
<column name="id" type="numeric" />
<column name="val" type="varchar" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_uuid">
<column name="id" type="numeric" />
<column name="val" type="varchar" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_date">
<column name="id" type="numeric" />
<column name="creation_date" type="Date" />
<column name="update_date" type="datetime" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_binary">
<column name="id" type="numeric" />
<column name="val" type="binary" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_varbinary">
<column name="id" type="numeric" />
<column name="val" type="varbinary" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_longblob">
<column name="id" type="numeric" />
<column name="val" type="longblob" />
</metadata>
<row data-node="expected_dataset.t_data_type_integer_unsigned" values="1001, 18446744073709551615, 4294967295, 16777215, 65535, 255" />
<row data-node="expected_dataset.t_data_type_integer_unsigned" values="1002, 0, 0, 0, 0, 0" />
<row data-node="expected_dataset.t_data_type_money" values="1001, 123" />
<row data-node="expected_dataset.t_data_type_money" values="1002, 456" />
<row data-node="expected_dataset.t_data_type_date" values="1, 2017-08-08, 2017-08-08 00:00:00" />
<row data-node="expected_dataset.t_data_type_binary" values="1, '123'" />
<row data-node="expected_dataset.t_data_type_varbinary" values="1, '123'" />
<row data-node="expected_dataset.t_data_type_longblob" values="1, '123'" />
</dataset>
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,6 @@ CREATE TABLE expected_dataset.t_data_type_money (id INT PRIMARY KEY, val NUMERI
CREATE TABLE expected_dataset.t_data_type_bytea (id INT PRIMARY KEY, val BLOB NOT NULL);
CREATE TABLE expected_dataset.t_data_type_date (id INT PRIMARY KEY, creation_date DATE NOT NULL, update_date DATETIME NOT NULL);
CREATE TABLE expected_dataset.t_data_type_uuid (id INT PRIMARY KEY, val VARCHAR(36) NOT NULL);
CREATE TABLE expected_dataset.t_data_type_binary (id INT PRIMARY KEY, val BINARY(10) NOT NULL);
CREATE TABLE expected_dataset.t_data_type_varbinary (id INT PRIMARY KEY, val VARBINARY(10) NOT NULL);
CREATE TABLE expected_dataset.t_data_type_longblob (id INT PRIMARY KEY, val LONGBLOB NOT NULL);

0 comments on commit 1f3af26

Please sign in to comment.