Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

MySQL JSON field type with string should decode in python3 #273

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

MySQL JSON field type with string should decode in python3 #273

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
eae3c26
Fix JSON field to decode bytes in python3
howmp Jul 10, 2018
fd82470
fix decode in python2
howmp Jul 10, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pymysqlreplication/packet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ def read_variable_length_string(self):
byte = byte2int(self.read(1))
length = length | ((byte & 0x7f) << bits_read)
bits_read = bits_read + 7
return self.read(length)
return self.read(length).decode('utf-8')

def read_int24(self):
a, b, c = struct.unpack("BBB", self.read(3))
Expand Down Expand Up @@ -429,7 +429,7 @@ def read_binary_json_object(self, length, large):
value_type_inlined_lengths = [read_offset_or_inline(self, large)
for _ in range(elements)]

keys = [self.read(x[1]) for x in key_offset_lengths]
keys = [self.read(x[1]).decode('utf-8') for x in key_offset_lengths]

out = {}
for i in range(elements):
Expand Down
23 changes: 6 additions & 17 deletions pymysqlreplication/tests/test_data_type.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,6 @@

__all__ = ["TestDataType"]


def to_binary_dict(d):
def encode_value(v):
if isinstance(v, text_type):
return v.encode()
if isinstance(v, list):
return [encode_value(x) for x in v]
return v
return dict([(k.encode(), encode_value(v)) for (k, v) in d.items()])


class TestDataType(base.PyMySQLReplicationTestCase):
def ignoredEvents(self):
return [GtidEvent]
Expand Down Expand Up @@ -431,15 +420,15 @@ def test_json(self):
create_query = "CREATE TABLE test (id int, value json);"
insert_query = """INSERT INTO test (id, value) VALUES (1, '{"my_key": "my_val", "my_key2": "my_val2"}');"""
event = self.create_and_insert_value(create_query, insert_query)
self.assertEqual(event.rows[0]["values"]["value"], {b"my_key": b"my_val", b"my_key2": b"my_val2"})
self.assertEqual(event.rows[0]["values"]["value"], {"my_key": "my_val", "my_key2": "my_val2"})

def test_json_array(self):
if not self.isMySQL57():
self.skipTest("Json is only supported in mysql 5.7")
create_query = "CREATE TABLE test (id int, value json);"
insert_query = """INSERT INTO test (id, value) VALUES (1, '["my_val", "my_val2"]');"""
event = self.create_and_insert_value(create_query, insert_query)
self.assertEqual(event.rows[0]["values"]["value"], [b'my_val', b'my_val2'])
self.assertEqual(event.rows[0]["values"]["value"], ['my_val', 'my_val2'])

def test_json_large(self):
if not self.isMySQL57():
Expand All @@ -449,7 +438,7 @@ def test_json_large(self):
insert_query = """INSERT INTO test (id, value) VALUES (1, '%s');""" % json.dumps(data)
event = self.create_and_insert_value(create_query, insert_query)

self.assertEqual(event.rows[0]["values"]["value"], to_binary_dict(data))
self.assertEqual(event.rows[0]["values"]["value"], data)

def test_json_types(self):
if not self.isMySQL57():
Expand All @@ -474,7 +463,7 @@ def test_json_types(self):
create_query = "CREATE TABLE test (id int, value json);"
insert_query = """INSERT INTO test (id, value) VALUES (1, '%s');""" % json.dumps(data)
event = self.create_and_insert_value(create_query, insert_query)
self.assertEqual(event.rows[0]["values"]["value"], to_binary_dict(data))
self.assertEqual(event.rows[0]["values"]["value"], data)

self.tearDown()
self.setUp()
Expand Down Expand Up @@ -511,7 +500,7 @@ def test_json_unicode(self):
create_query = "CREATE TABLE test (id int, value json);"
insert_query = u"""INSERT INTO test (id, value) VALUES (1, '{"miam": "🍔"}');"""
event = self.create_and_insert_value(create_query, insert_query)
self.assertEqual(event.rows[0]["values"]["value"][b"miam"], u'🍔'.encode('utf8'))
self.assertEqual(event.rows[0]["values"]["value"]["miam"], u'🍔')

def test_json_long_string(self):
if not self.isMySQL57():
Expand All @@ -521,7 +510,7 @@ def test_json_long_string(self):
string_value = "super_long_string" * 100
insert_query = "INSERT INTO test (id, value) VALUES (1, '{\"my_key\": \"%s\"}');" % (string_value,)
event = self.create_and_insert_value(create_query, insert_query)
self.assertEqual(event.rows[0]["values"]["value"], to_binary_dict({"my_key": string_value}))
self.assertEqual(event.rows[0]["values"]["value"], {"my_key": string_value})

def test_null(self):
create_query = "CREATE TABLE test ( \
Expand Down