From 67120c7c3eb9e40ba682aeb150f70b6eaab0d494 Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Sun, 1 Dec 2024 17:14:03 +0100 Subject: [PATCH 1/2] feat: enable view arrow export --- py-polars/polars/dataframe/frame.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/py-polars/polars/dataframe/frame.py b/py-polars/polars/dataframe/frame.py index 649b1b15120e..92b10e81ca45 100644 --- a/py-polars/polars/dataframe/frame.py +++ b/py-polars/polars/dataframe/frame.py @@ -4283,13 +4283,18 @@ def write_delta( _check_if_delta_available() from deltalake import DeltaTable, write_deltalake + from deltalake import __version__ as delta_version + from packaging.version import Version _check_for_unsupported_types(self.dtypes) if isinstance(target, (str, Path)): target = _resolve_delta_lake_uri(str(target), strict=False) - data = self.to_arrow() + if Version(delta_version) >= Version("0.22.3"): + data = self.to_arrow(compat_level=CompatLevel.newest()) + else: + data = self.to_arrow() if mode == "merge": if delta_merge_options is None: @@ -4316,7 +4321,6 @@ def write_delta( schema=schema, mode=mode, storage_options=storage_options, - large_dtypes=True, **delta_write_options, ) return None From 53a233b2c05571a4d9e7912b1f7c921afbefa20e Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Mon, 2 Dec 2024 09:55:34 +0100 Subject: [PATCH 2/2] chore: increase deltalake minimum --- py-polars/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py-polars/pyproject.toml b/py-polars/pyproject.toml index 63cf77976175..fa060fb48810 100644 --- a/py-polars/pyproject.toml +++ b/py-polars/pyproject.toml @@ -61,7 +61,7 @@ database = ["polars[adbc,connectorx,sqlalchemy]", "nest-asyncio"] fsspec = ["fsspec"] # Other I/O -deltalake = ["deltalake >= 0.15.0"] +deltalake = ["deltalake >= 0.19.0"] iceberg = ["pyiceberg >= 0.5.0"] # Other