From 06e90d13608d45107d8d07577fae0af2d6436007 Mon Sep 17 00:00:00 2001 From: brandonzhang Date: Sat, 6 Jul 2024 17:21:43 +0800 Subject: [PATCH 1/2] read mysql bit type as i64 --- connectorx/src/sources/mysql/typesystem.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/connectorx/src/sources/mysql/typesystem.rs b/connectorx/src/sources/mysql/typesystem.rs index 105b7271ca..c0ec97cd73 100644 --- a/connectorx/src/sources/mysql/typesystem.rs +++ b/connectorx/src/sources/mysql/typesystem.rs @@ -116,6 +116,7 @@ impl<'a> From<(&'a ColumnType, &'a ColumnFlags)> for MySQLTypeSystem { ColumnType::MYSQL_TYPE_LONG_BLOB => LongBlob(null_ok), ColumnType::MYSQL_TYPE_JSON => Json(null_ok), ColumnType::MYSQL_TYPE_VARCHAR => VarChar(null_ok), + ColumnType::MYSQL_TYPE_BIT => LongLong(null_ok), _ => unimplemented!("{}", format!("{:?}", ty)), } } From 4e14d32cb1ff91a9b825bd4e3a6f6d8e95f1815f Mon Sep 17 00:00:00 2001 From: Xiaoying Wang Date: Wed, 2 Oct 2024 15:25:41 -0700 Subject: [PATCH 2/2] add test & fix --- connectorx-python/connectorx/tests/test_mysql.py | 6 ++++++ connectorx-python/src/pandas/transports/mysql.rs | 2 ++ connectorx/src/sources/mysql/typesystem.rs | 5 +++-- connectorx/src/transports/mysql_arrow.rs | 2 ++ connectorx/src/transports/mysql_arrow2.rs | 2 ++ connectorx/src/transports/mysql_arrowstream.rs | 2 ++ scripts/mysql.sql | 9 +++++---- 7 files changed, 22 insertions(+), 6 deletions(-) diff --git a/connectorx-python/connectorx/tests/test_mysql.py b/connectorx-python/connectorx/tests/test_mysql.py index 51ffcf1683..d76b56305e 100644 --- a/connectorx-python/connectorx/tests/test_mysql.py +++ b/connectorx-python/connectorx/tests/test_mysql.py @@ -337,6 +337,9 @@ def test_mysql_types_binary(mysql_url: str) -> None: "test_mediumtext": pd.Series( [None, b"", b"medium text!!!!"], dtype="object" ), + "test_bit": pd.Series( + [b'\x17', b'\x18', None] + ), }, ) assert_frame_equal(df, expected, check_names=True) @@ -406,6 +409,9 @@ def test_mysql_types_text(mysql_url: str) -> None: "test_mediumtext": pd.Series( [None, b"", b"medium text!!!!"], dtype="object" ), + "test_bit": pd.Series( + [b'\x17', b'\x18', None] + ), }, ) assert_frame_equal(df, expected, check_names=True) diff --git a/connectorx-python/src/pandas/transports/mysql.rs b/connectorx-python/src/pandas/transports/mysql.rs index 3f853b6c16..3cf175c71f 100644 --- a/connectorx-python/src/pandas/transports/mysql.rs +++ b/connectorx-python/src/pandas/transports/mysql.rs @@ -46,6 +46,7 @@ impl_transport!( { MediumBlob[Vec] => Bytes[Vec] | conversion none } { LongBlob[Vec] => Bytes[Vec] | conversion none } { Json[Value] => String[String] | conversion option } + { Bit[Vec] => Bytes[Vec] | conversion none } } ); @@ -81,6 +82,7 @@ impl_transport!( { MediumBlob[Vec] => Bytes[Vec] | conversion none } { LongBlob[Vec] => Bytes[Vec] | conversion none } { Json[Value] => String[String] | conversion option } + { Bit[Vec] => Bytes[Vec] | conversion none } } ); diff --git a/connectorx/src/sources/mysql/typesystem.rs b/connectorx/src/sources/mysql/typesystem.rs index c0ec97cd73..f2cdcd0e2c 100644 --- a/connectorx/src/sources/mysql/typesystem.rs +++ b/connectorx/src/sources/mysql/typesystem.rs @@ -31,6 +31,7 @@ pub enum MySQLTypeSystem { MediumBlob(bool), LongBlob(bool), Json(bool), + Bit(bool), } impl_typesystem! { @@ -51,7 +52,7 @@ impl_typesystem! { { Time => NaiveTime } { Decimal => Decimal } { Char | VarChar | Enum => String } - { TinyBlob | Blob | MediumBlob | LongBlob => Vec} + { TinyBlob | Blob | MediumBlob | LongBlob | Bit => Vec} { Json => Value } } } @@ -116,7 +117,7 @@ impl<'a> From<(&'a ColumnType, &'a ColumnFlags)> for MySQLTypeSystem { ColumnType::MYSQL_TYPE_LONG_BLOB => LongBlob(null_ok), ColumnType::MYSQL_TYPE_JSON => Json(null_ok), ColumnType::MYSQL_TYPE_VARCHAR => VarChar(null_ok), - ColumnType::MYSQL_TYPE_BIT => LongLong(null_ok), + ColumnType::MYSQL_TYPE_BIT => Bit(null_ok), _ => unimplemented!("{}", format!("{:?}", ty)), } } diff --git a/connectorx/src/transports/mysql_arrow.rs b/connectorx/src/transports/mysql_arrow.rs index 05675a6f32..01d15df853 100644 --- a/connectorx/src/transports/mysql_arrow.rs +++ b/connectorx/src/transports/mysql_arrow.rs @@ -65,6 +65,7 @@ impl_transport!( { MediumBlob[Vec] => LargeBinary[Vec] | conversion none } { LongBlob[Vec] => LargeBinary[Vec] | conversion none } { Json[Value] => LargeUtf8[String] | conversion option } + { Bit[Vec] => LargeBinary[Vec] | conversion none } } ); @@ -100,6 +101,7 @@ impl_transport!( { MediumBlob[Vec] => LargeBinary[Vec] | conversion none } { LongBlob[Vec] => LargeBinary[Vec] | conversion none } { Json[Value] => LargeUtf8[String] | conversion option } + { Bit[Vec] => LargeBinary[Vec] | conversion none } } ); diff --git a/connectorx/src/transports/mysql_arrow2.rs b/connectorx/src/transports/mysql_arrow2.rs index f239e3b32d..2e1c57c795 100644 --- a/connectorx/src/transports/mysql_arrow2.rs +++ b/connectorx/src/transports/mysql_arrow2.rs @@ -65,6 +65,7 @@ impl_transport!( { Blob[Vec] => LargeBinary[Vec] | conversion none } { MediumBlob[Vec] => LargeBinary[Vec] | conversion none } { LongBlob[Vec] => LargeBinary[Vec] | conversion none } + { Bit[Vec] => LargeBinary[Vec] | conversion none } } ); @@ -100,6 +101,7 @@ impl_transport!( { Blob[Vec] => LargeBinary[Vec] | conversion none } { MediumBlob[Vec] => LargeBinary[Vec] | conversion none } { LongBlob[Vec] => LargeBinary[Vec] | conversion none } + { Bit[Vec] => LargeBinary[Vec] | conversion none } } ); diff --git a/connectorx/src/transports/mysql_arrowstream.rs b/connectorx/src/transports/mysql_arrowstream.rs index 0d0bccd76e..94542e2a8a 100644 --- a/connectorx/src/transports/mysql_arrowstream.rs +++ b/connectorx/src/transports/mysql_arrowstream.rs @@ -64,6 +64,7 @@ impl_transport!( { MediumBlob[Vec] => LargeBinary[Vec] | conversion none } { LongBlob[Vec] => LargeBinary[Vec] | conversion none } { Json[Value] => LargeUtf8[String] | conversion option } + { Bit[Vec] => LargeBinary[Vec] | conversion none } } ); @@ -99,6 +100,7 @@ impl_transport!( { MediumBlob[Vec] => LargeBinary[Vec] | conversion none } { LongBlob[Vec] => LargeBinary[Vec] | conversion none } { Json[Value] => LargeUtf8[String] | conversion option } + { Bit[Vec] => LargeBinary[Vec] | conversion none } } ); diff --git a/scripts/mysql.sql b/scripts/mysql.sql index 7906415a3c..7d86576709 100644 --- a/scripts/mysql.sql +++ b/scripts/mysql.sql @@ -61,9 +61,10 @@ CREATE TABLE IF NOT EXISTS test_types( test_longblob LONGBLOB, test_enum ENUM('apple', 'banana', 'orange', 'mango'), test_json JSON, - test_mediumtext MEDIUMTEXT + test_mediumtext MEDIUMTEXT, + test_bit BIT(5) ); -INSERT INTO test_types VALUES ('1970-01-01 00:00:01', NULL, '00:00:00', '1970-01-01 00:00:01', 1.1, 1, NULL, 'char1', -128, -32768, -8388608, -2147483648, -9223372036854775808, NULL, NULL, NULL, NULL, NULL, 1, 1, NULL, -2.2E-308, 1.2345, 1901, NULL, NULL, NULL, NULL, 'apple', '{"name": "piggy", "age": 1}', NULL); -INSERT INTO test_types VALUES ('2038-01-19 00:00:00', '1970-01-01', NULL, '2038-01-19 00:0:00', NULL, 2, 'varchar2', NULL, 127, 32767, 8388607, 2147483647, 9223372036854775807, 255, 65535, 16777215, 4294967295, 1.844674407E19, 2147483647, 65535, -1.1E-38, NULL, -1.1E-3, 2155, 'tinyblob2', 'blobblobblobblob2', 'mediumblob2', 'longblob2', NULL, '{"name": "kitty", "age": 2}', ''); -INSERT INTO test_types VALUES (NULL, '2038-01-19', '23:59:59', NULL, 3.3, NULL, 'varchar3', 'char3', NULL, NULL, NULL, NULL, NULL, 0, 0, 0, 0, 0, -2147483648, 0, 3.4E38, 1.7E308, 1.7E30, NULL, 'tinyblob3', 'blobblobblobblob3', 'mediumblob3', 'longblob3', 'mango', NULL, 'medium text!!!!'); \ No newline at end of file +INSERT INTO test_types VALUES ('1970-01-01 00:00:01', NULL, '00:00:00', '1970-01-01 00:00:01', 1.1, 1, NULL, 'char1', -128, -32768, -8388608, -2147483648, -9223372036854775808, NULL, NULL, NULL, NULL, NULL, 1, 1, NULL, -2.2E-308, 1.2345, 1901, NULL, NULL, NULL, NULL, 'apple', '{"name": "piggy", "age": 1}', NULL, b'10111'); +INSERT INTO test_types VALUES ('2038-01-19 00:00:00', '1970-01-01', NULL, '2038-01-19 00:0:00', NULL, 2, 'varchar2', NULL, 127, 32767, 8388607, 2147483647, 9223372036854775807, 255, 65535, 16777215, 4294967295, 1.844674407E19, 2147483647, 65535, -1.1E-38, NULL, -1.1E-3, 2155, 'tinyblob2', 'blobblobblobblob2', 'mediumblob2', 'longblob2', NULL, '{"name": "kitty", "age": 2}', '', b'11000'); +INSERT INTO test_types VALUES (NULL, '2038-01-19', '23:59:59', NULL, 3.3, NULL, 'varchar3', 'char3', NULL, NULL, NULL, NULL, NULL, 0, 0, 0, 0, 0, -2147483648, 0, 3.4E38, 1.7E308, 1.7E30, NULL, 'tinyblob3', 'blobblobblobblob3', 'mediumblob3', 'longblob3', 'mango', NULL, 'medium text!!!!', NULL); \ No newline at end of file