From 7597ffa6803ec71960569d2e1c772a5fde2f1722 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Fri, 17 Nov 2023 15:30:29 -0500 Subject: [PATCH 1/3] Update pyarrow dependency --- python-spec/requirements-py3.10.txt | 2 +- python-spec/requirements-py3.7.txt | 2 +- python-spec/requirements-py3.8-lint.txt | 2 +- python-spec/requirements-py3.8.txt | 2 +- python-spec/requirements-py3.9.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python-spec/requirements-py3.10.txt b/python-spec/requirements-py3.10.txt index 634cc44f..cb49783e 100644 --- a/python-spec/requirements-py3.10.txt +++ b/python-spec/requirements-py3.10.txt @@ -7,7 +7,7 @@ numba==0.58.0 numpy==1.25.2 packaging==23.1 pandas==2.1.1 -pyarrow==13.0.0 +pyarrow>=14.0.1 python-dateutil==2.8.2 pytz==2023.3.post1 scipy==1.11.3 diff --git a/python-spec/requirements-py3.7.txt b/python-spec/requirements-py3.7.txt index b7d5617d..40007d5d 100644 --- a/python-spec/requirements-py3.7.txt +++ b/python-spec/requirements-py3.7.txt @@ -8,7 +8,7 @@ numba==0.56.4 numpy==1.21.6 packaging==23.1 pandas==1.3.5 -pyarrow==12.0.1 +pyarrow!=14.0.0 python-dateutil==2.8.2 pytz==2023.3.post1 scipy==1.7.3 diff --git a/python-spec/requirements-py3.8-lint.txt b/python-spec/requirements-py3.8-lint.txt index b5d21a14..f0ba4f3a 100644 --- a/python-spec/requirements-py3.8-lint.txt +++ b/python-spec/requirements-py3.8-lint.txt @@ -21,7 +21,7 @@ pandas==2.0.3 pathspec==0.11.2 platformdirs==3.10.0 pre-commit==3.4.0 -pyarrow==13.0.0 +pyarrow>=14.0.1 python-dateutil==2.8.2 pytz==2023.3.post1 PyYAML==6.0.1 diff --git a/python-spec/requirements-py3.8.txt b/python-spec/requirements-py3.8.txt index a652c0de..12191709 100644 --- a/python-spec/requirements-py3.8.txt +++ b/python-spec/requirements-py3.8.txt @@ -8,7 +8,7 @@ numba==0.58.0 numpy==1.24.4 packaging==23.1 pandas==2.0.3 -pyarrow==13.0.0 +pyarrow>=14.0.1 python-dateutil==2.8.2 pytz==2023.3.post1 scipy==1.10.1 diff --git a/python-spec/requirements-py3.9.txt b/python-spec/requirements-py3.9.txt index 194ec037..6a6c07e0 100644 --- a/python-spec/requirements-py3.9.txt +++ b/python-spec/requirements-py3.9.txt @@ -9,7 +9,7 @@ numba==0.58.0 numpy==1.25.2 packaging==23.1 pandas==2.1.1 -pyarrow==13.0.0 +pyarrow>=14.0.1 python-dateutil==2.8.2 pytz==2023.3.post1 rsa==4.7.2 From af9f364c1db57084454a231899f6d6c8c3f57d8a Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 21 Nov 2023 16:06:39 -0500 Subject: [PATCH 2/3] Use pyarrow_hotfix --- pyproject.toml | 5 ++++- python-spec/requirements-py3.10.txt | 3 ++- python-spec/requirements-py3.7.txt | 3 ++- python-spec/requirements-py3.8-lint.txt | 3 ++- python-spec/requirements-py3.8.txt | 3 ++- python-spec/requirements-py3.9.txt | 3 ++- python-spec/src/somacore/collection.py | 4 ++++ python-spec/src/somacore/data.py | 3 +++ python-spec/src/somacore/options.py | 3 +++ python-spec/src/somacore/query/_fast_csr.py | 3 +++ python-spec/src/somacore/query/axis.py | 3 +++ python-spec/src/somacore/query/query.py | 3 +++ 12 files changed, 33 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d4a6fd21..843a2aaa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,9 @@ dependencies = [ "numpy>=1.21", "pandas", "pyarrow", + # TODO: once we no longer support Python 3.7, remove this and pin to pyarrow >= 14.0.1 + # https://github.com/single-cell-data/TileDB-SOMA/issues/1926 + "pyarrow-hotfix", "scipy", "typing-extensions>=4.1", # For LiteralString ] @@ -56,5 +59,5 @@ warn_redundant_casts = true [[tool.mypy.overrides]] # These dependencies do not currently have canonical type stubs. -module = ["anndata", "numba", "pandas", "pyarrow", "scipy"] +module = ["anndata", "numba", "pandas", "pyarrow", "pyarrow_hotfix", "scipy"] ignore_missing_imports = true diff --git a/python-spec/requirements-py3.10.txt b/python-spec/requirements-py3.10.txt index cb49783e..8958867a 100644 --- a/python-spec/requirements-py3.10.txt +++ b/python-spec/requirements-py3.10.txt @@ -7,7 +7,8 @@ numba==0.58.0 numpy==1.25.2 packaging==23.1 pandas==2.1.1 -pyarrow>=14.0.1 +pyarrow==13.0.0 +pyarrow_hotfix==0.6.0 python-dateutil==2.8.2 pytz==2023.3.post1 scipy==1.11.3 diff --git a/python-spec/requirements-py3.7.txt b/python-spec/requirements-py3.7.txt index 40007d5d..cd13e212 100644 --- a/python-spec/requirements-py3.7.txt +++ b/python-spec/requirements-py3.7.txt @@ -8,7 +8,8 @@ numba==0.56.4 numpy==1.21.6 packaging==23.1 pandas==1.3.5 -pyarrow!=14.0.0 +pyarrow==12.0.1 +pyarrow_hotfix==0.6.0 python-dateutil==2.8.2 pytz==2023.3.post1 scipy==1.7.3 diff --git a/python-spec/requirements-py3.8-lint.txt b/python-spec/requirements-py3.8-lint.txt index f0ba4f3a..d20d9883 100644 --- a/python-spec/requirements-py3.8-lint.txt +++ b/python-spec/requirements-py3.8-lint.txt @@ -21,7 +21,8 @@ pandas==2.0.3 pathspec==0.11.2 platformdirs==3.10.0 pre-commit==3.4.0 -pyarrow>=14.0.1 +pyarrow==13.0.0 +pyarrow_hotfix==0.6.0 python-dateutil==2.8.2 pytz==2023.3.post1 PyYAML==6.0.1 diff --git a/python-spec/requirements-py3.8.txt b/python-spec/requirements-py3.8.txt index 12191709..e44bafe3 100644 --- a/python-spec/requirements-py3.8.txt +++ b/python-spec/requirements-py3.8.txt @@ -8,7 +8,8 @@ numba==0.58.0 numpy==1.24.4 packaging==23.1 pandas==2.0.3 -pyarrow>=14.0.1 +pyarrow==13.0.0 +pyarrow_hotfix==0.6.0 python-dateutil==2.8.2 pytz==2023.3.post1 scipy==1.10.1 diff --git a/python-spec/requirements-py3.9.txt b/python-spec/requirements-py3.9.txt index 6a6c07e0..e2e95445 100644 --- a/python-spec/requirements-py3.9.txt +++ b/python-spec/requirements-py3.9.txt @@ -9,7 +9,8 @@ numba==0.58.0 numpy==1.25.2 packaging==23.1 pandas==2.1.1 -pyarrow>=14.0.1 +pyarrow==13.0.0 +pyarrow_hotfix==0.6.0 python-dateutil==2.8.2 pytz==2023.3.post1 rsa==4.7.2 diff --git a/python-spec/src/somacore/collection.py b/python-spec/src/somacore/collection.py index 1ccec0a4..19f2e2a6 100644 --- a/python-spec/src/somacore/collection.py +++ b/python-spec/src/somacore/collection.py @@ -1,7 +1,11 @@ import abc from typing import Any, MutableMapping, Optional, Sequence, Type, TypeVar, overload +# ruff: noqa import pyarrow as pa + +# ruff: noqa +import pyarrow_hotfix from typing_extensions import Final, Self from . import base diff --git a/python-spec/src/somacore/data.py b/python-spec/src/somacore/data.py index e8bffd66..86f49d9d 100644 --- a/python-spec/src/somacore/data.py +++ b/python-spec/src/somacore/data.py @@ -19,6 +19,9 @@ ) import pyarrow as pa + +# ruff: noqa +import pyarrow_hotfix from typing_extensions import Final, Literal, Self from . import base diff --git a/python-spec/src/somacore/options.py b/python-spec/src/somacore/options.py index 4e09ca37..f562736f 100644 --- a/python-spec/src/somacore/options.py +++ b/python-spec/src/somacore/options.py @@ -11,6 +11,9 @@ import numpy as np import numpy.typing as npt import pyarrow as pa + +# ruff: noqa +import pyarrow_hotfix from typing_extensions import Final, Literal from . import types diff --git a/python-spec/src/somacore/query/_fast_csr.py b/python-spec/src/somacore/query/_fast_csr.py index 17fb0aa2..8140c803 100644 --- a/python-spec/src/somacore/query/_fast_csr.py +++ b/python-spec/src/somacore/query/_fast_csr.py @@ -8,6 +8,9 @@ import numpy.typing as npt import pandas as pd import pyarrow as pa + +# ruff: noqa +import pyarrow_hotfix from scipy import sparse from .. import data as scd diff --git a/python-spec/src/somacore/query/axis.py b/python-spec/src/somacore/query/axis.py index 7ae660e8..1a5eabc7 100644 --- a/python-spec/src/somacore/query/axis.py +++ b/python-spec/src/somacore/query/axis.py @@ -4,6 +4,9 @@ import numpy as np import pyarrow as pa +# ruff: noqa +import pyarrow_hotfix + from .. import options from .. import types diff --git a/python-spec/src/somacore/query/query.py b/python-spec/src/somacore/query/query.py index 1032517e..d51f15c4 100644 --- a/python-spec/src/somacore/query/query.py +++ b/python-spec/src/somacore/query/query.py @@ -18,6 +18,9 @@ import numpy.typing as npt import pandas as pd import pyarrow as pa + +# ruff: noqa +import pyarrow_hotfix from scipy import sparse from typing_extensions import Literal, Protocol, Self, TypedDict, assert_never From 1c2b483ba4a0067bf83b5312612d96812b922d24 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 21 Nov 2023 16:12:36 -0500 Subject: [PATCH 3/3] round n+1 of code review --- python-spec/src/somacore/__init__.py | 5 +++++ python-spec/src/somacore/collection.py | 4 ---- python-spec/src/somacore/data.py | 3 --- python-spec/src/somacore/options.py | 3 --- python-spec/src/somacore/query/_fast_csr.py | 3 --- python-spec/src/somacore/query/axis.py | 3 --- python-spec/src/somacore/query/query.py | 3 --- 7 files changed, 5 insertions(+), 19 deletions(-) diff --git a/python-spec/src/somacore/__init__.py b/python-spec/src/somacore/__init__.py index 21b0c941..2cce016d 100644 --- a/python-spec/src/somacore/__init__.py +++ b/python-spec/src/somacore/__init__.py @@ -9,6 +9,11 @@ from typing import Tuple, Union +# TODO: once we no longer support Python 3.7, remove this and pin to pyarrow >= 14.0.1 +# https://github.com/single-cell-data/TileDB-SOMA/issues/1926 +# ruff: noqa +import pyarrow_hotfix + from .base import SOMAObject from .collection import Collection from .data import DataFrame diff --git a/python-spec/src/somacore/collection.py b/python-spec/src/somacore/collection.py index 19f2e2a6..1ccec0a4 100644 --- a/python-spec/src/somacore/collection.py +++ b/python-spec/src/somacore/collection.py @@ -1,11 +1,7 @@ import abc from typing import Any, MutableMapping, Optional, Sequence, Type, TypeVar, overload -# ruff: noqa import pyarrow as pa - -# ruff: noqa -import pyarrow_hotfix from typing_extensions import Final, Self from . import base diff --git a/python-spec/src/somacore/data.py b/python-spec/src/somacore/data.py index 86f49d9d..e8bffd66 100644 --- a/python-spec/src/somacore/data.py +++ b/python-spec/src/somacore/data.py @@ -19,9 +19,6 @@ ) import pyarrow as pa - -# ruff: noqa -import pyarrow_hotfix from typing_extensions import Final, Literal, Self from . import base diff --git a/python-spec/src/somacore/options.py b/python-spec/src/somacore/options.py index f562736f..4e09ca37 100644 --- a/python-spec/src/somacore/options.py +++ b/python-spec/src/somacore/options.py @@ -11,9 +11,6 @@ import numpy as np import numpy.typing as npt import pyarrow as pa - -# ruff: noqa -import pyarrow_hotfix from typing_extensions import Final, Literal from . import types diff --git a/python-spec/src/somacore/query/_fast_csr.py b/python-spec/src/somacore/query/_fast_csr.py index 8140c803..17fb0aa2 100644 --- a/python-spec/src/somacore/query/_fast_csr.py +++ b/python-spec/src/somacore/query/_fast_csr.py @@ -8,9 +8,6 @@ import numpy.typing as npt import pandas as pd import pyarrow as pa - -# ruff: noqa -import pyarrow_hotfix from scipy import sparse from .. import data as scd diff --git a/python-spec/src/somacore/query/axis.py b/python-spec/src/somacore/query/axis.py index 1a5eabc7..7ae660e8 100644 --- a/python-spec/src/somacore/query/axis.py +++ b/python-spec/src/somacore/query/axis.py @@ -4,9 +4,6 @@ import numpy as np import pyarrow as pa -# ruff: noqa -import pyarrow_hotfix - from .. import options from .. import types diff --git a/python-spec/src/somacore/query/query.py b/python-spec/src/somacore/query/query.py index d51f15c4..1032517e 100644 --- a/python-spec/src/somacore/query/query.py +++ b/python-spec/src/somacore/query/query.py @@ -18,9 +18,6 @@ import numpy.typing as npt import pandas as pd import pyarrow as pa - -# ruff: noqa -import pyarrow_hotfix from scipy import sparse from typing_extensions import Literal, Protocol, Self, TypedDict, assert_never