Skip to content

Commit

Permalink
Introduce Polars LazyFrame serialization formats (#483)
Browse files Browse the repository at this point in the history
  • Loading branch information
JanusAsmussen authored Oct 7, 2024
1 parent 8e6110c commit 2b6b234
Show file tree
Hide file tree
Showing 5 changed files with 942 additions and 812 deletions.
1 change: 1 addition & 0 deletions .pylintrc
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
# it should appear only once).
disable=W0212,R0903,C0301,R0902

max-positional-arguments = 15
max-args = 15
max-locals = 20
min-similarity-lines = 20
69 changes: 69 additions & 0 deletions adapta/storage/models/format.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,75 @@ def deserialize(self, data: bytes) -> polars.DataFrame:
return polars.read_json(io.BytesIO(data))


class PolarsLazyFrameParquetSerializationFormat(SerializationFormat[polars.LazyFrame]):
"""
Serializes lazyframes as parquet format.
"""

def serialize(self, data: polars.LazyFrame) -> bytes:
"""
Serializes lazyframe to bytes using parquet format.
:param data: Lazyframe to serialize.
:return: Parquet serialized lazyframe as byte array.
"""
buffer = io.BytesIO()
data.collect().write_parquet(buffer)
return buffer.getvalue()

def deserialize(self, data: bytes) -> polars.LazyFrame:
"""
Deserializes lazyframe from bytes using parquet format.
:param data: Lazyframe to deserialize in parquet format as bytes.
:return: Deserialized lazyframe.
"""
return polars.scan_parquet(io.BytesIO(data))


class PolarsLazyFrameCsvSerializationFormat(SerializationFormat[polars.LazyFrame]):
"""
Serializes lazyframes as CSV format.
"""

def serialize(self, data: polars.LazyFrame) -> bytes:
"""
Serializes lazyframe to bytes using CSV format.
:param data: Lazyframe to serialize.
:return: CSV serialized Lazyframe as byte array.
"""

return data.collect().write_csv().encode(encoding="utf-8")

def deserialize(self, data: bytes) -> polars.LazyFrame:
"""
Deserializes lazyframe from bytes using CSV format.
:param data: LazyFrame to deserialize in CSV format as bytes.
:return: Deserialized lazyframe.
"""
return polars.scan_csv(io.BytesIO(data))


class PolarsLazyFrameJsonSerializationFormat(SerializationFormat[polars.LazyFrame]):
"""
Serializes lazyframes as JSON format.
"""

def serialize(self, data: polars.LazyFrame) -> bytes:
"""
Serializes lazyframes to bytes using JSON format.
:param data: LazyFrame to serialize.
:return: JSON serialized lazyframe as byte array.
"""
return data.collect().write_ndjson().encode(encoding="utf-8")

def deserialize(self, data: bytes) -> polars.LazyFrame:
"""
Deserializes lazyframes from bytes using JSON format.
:param data: LazyFrame to deserialize in JSON format as bytes.
:return: Deserialized lazyframe.
"""
return polars.scan_ndjson(io.BytesIO(data))


class DictJsonSerializationFormat(SerializationFormat[dict]):
"""
Serializes dictionaries as JSON format.
Expand Down
Loading

0 comments on commit 2b6b234

Please sign in to comment.