Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix dataframe (de)serialization #600

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open

Fix dataframe (de)serialization #600

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
d90d216
Add failing test with a dataframe that has datetimes in it
Feb 3, 2022
d8a4c8c
Add new dataframe ser/de implementations, tests pass
Feb 3, 2022
8a00687
Add load from array test that breaks pandas
Feb 3, 2022
a266106
Fix operation for matrix-defined dataframes, allow dates to serialize…
Feb 3, 2022
18882d4
Add verification of datetime serialization roundtrips to pandas test
Feb 3, 2022
f59a022
Remove redundant "if" statement
Feb 4, 2022
5501d0e
Globally import json instead of inline
Feb 4, 2022
4bc0c50
Merge branch 'main' into pr/ektar/600
maximlt Apr 12, 2023
7e5098c
update tests
maximlt Apr 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions param/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1567,14 +1567,16 @@ def _validate(self, val):
def serialize(cls, value):
if value is None:
return 'null'
return value.to_dict('records')
import json
ektar marked this conversation as resolved.
Show resolved Hide resolved
return json.loads(value.to_json(date_format='iso'))

@classmethod
def deserialize(cls, value):
if value == 'null':
return None
from pandas import DataFrame as pdDFrame
return pdDFrame(value)
import json
import pandas as pd
return pd.read_json(json.dumps(value))


class Series(ClassSelector):
Expand Down
19 changes: 17 additions & 2 deletions tests/API1/testjsonserialization.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ class TestSet(param.Parameterized):
__test__ = False

numpy_params = ['r','y']
pandas_params = ['s','t','u','z']
pandas_params = ['s','t','u','z','ab','ac']
conditionally_unsafe = ['f', 'o']

a = param.Integer(default=5, doc='Example doc', bounds=(2,30), inclusive_bounds=(True, False))
Expand Down Expand Up @@ -73,6 +73,12 @@ class TestSet(param.Parameterized):
y = None if np is None else param.Array(default=None)
z = None if pd is None else param.DataFrame(default=None, allow_None=True)
aa = param.Tuple(default=None, allow_None=True, length=1)
ab = None if pd is None else param.DataFrame(default=pd.DataFrame(
{'A':[datetime.datetime(year,1,1) for year in range(2020,2023)], 'B':[1.1,2.2,3.3]}),
columns=(1,4), rows=(2,5))
ac = None if pd is None else param.DataFrame(default=pd.DataFrame(
[[1,2,3],[4,5,6],[7,8,9]]),
columns=(1,4), rows=(2,5))


test = TestSet(a=29)
Expand Down Expand Up @@ -186,13 +192,22 @@ def test_numpy_instance_serialization(self):

@pd_skip
def test_pandas_instance_serialization(self):
from pandas.api.types import is_datetime64_any_dtype as is_datetime
serialized = test.param.serialize_parameters(subset=test.pandas_params, mode=self.mode)
deserialized = TestSet.param.deserialize_parameters(serialized, mode=self.mode)
for pname in test.pandas_params:
if getattr(test, pname) is None:
self.assertTrue(deserialized[pname] is None)
else:
self.assertTrue(getattr(test, pname).equals(deserialized[pname]))
test_df = getattr(test, pname)
deser_df = deserialized[pname].copy()

date_cols = [c for c in test_df.columns if is_datetime(test_df[c])]
for c in date_cols:
src_tz = test_df.loc[0, c].tz
deser_df[c] = pd.to_datetime(deser_df[c]).dt.tz_convert(src_tz)

self.assertTrue(test_df.equals(deser_df))



Expand Down