Skip to content

Commit

Permalink
linting
Browse files Browse the repository at this point in the history
  • Loading branch information
ghidalgo3 committed Jul 16, 2024
1 parent 5d9717a commit 54650ac
Show file tree
Hide file tree
Showing 4 changed files with 14 additions and 3 deletions.
2 changes: 1 addition & 1 deletion kerchunk/fits.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def process_file(
inline_threshold=100,
primary_attr_to_group=False,
out=None,
zarr_version=None
zarr_version=None,
):
"""
Create JSON references for a single FITS file as a zarr group
Expand Down
3 changes: 3 additions & 0 deletions kerchunk/tests/test_fits.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
range_im = os.path.join(testdir, "arange.fits")
var = os.path.join(testdir, "variable_length_table.fits")


@pytest.mark.parametrize("zarr_version", [2, 3])
def test_ascii_table(zarr_version):
# this one directly hits a remote server - should cache?
Expand Down Expand Up @@ -57,6 +58,7 @@ def test_cube(zarr_version):
expected = hdul[0].data
assert (arr[:] == expected).all()


@pytest.mark.parametrize("zarr_version", [2, 3])
def test_with_class(zarr_version):
ftz = kerchunk.fits.FitsToZarr(range_im)
Expand All @@ -70,6 +72,7 @@ def test_with_class(zarr_version):
expected = hdul[0].data
assert (arr[:] == expected).all()


@pytest.mark.parametrize("zarr_version", [2, 3])
def test_var(zarr_version):
data = fits.open(var)[1].data
Expand Down
1 change: 1 addition & 0 deletions kerchunk/tests/test_hdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

here = osp.dirname(__file__)


@pytest.mark.parametrize("zarr_version", [2, 3])
def test_single(zarr_version):
"""Test creating references for a single HDF file"""
Expand Down
11 changes: 9 additions & 2 deletions kerchunk/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@
from zarr.store import StorePath, MemoryStore
from zarr.v2.hierarchy import group
import zarr.array

_ZARR_VERSION = 3
except:
except ModuleNotFoundError:
_ZARR_VERSION = 2



def class_factory(func):
"""Experimental uniform API across function-based file scanners"""

Expand Down Expand Up @@ -62,17 +62,21 @@ def consolidate(refs):
out[k] = v
return {"version": 1, "refs": out}


def encode_fill_value(v, dtype, object_codec=None):
if _ZARR_VERSION == 3:
# Precarious use of this function
# https://github.com/zarr-developers/zarr-python/issues/2021
# https://github.com/zarr-developers/VirtualiZarr/pull/182#discussion_r1673096418
from zarr.v2.meta import Metadata2

return Metadata2.encode_fill_value(v, dtype, object_codec)
else:
from zarr.meta import encode_fill_value as _encode_fill_value

return _encode_fill_value(v, dtype, object_codec)


def rename_target(refs, renames):
"""Utility to change URLs in a reference set in a predictable way
Expand Down Expand Up @@ -135,6 +139,7 @@ def rename_target_files(
with fsspec.open(url_out, mode="wt", **(storage_options_out or {})) as f:
ujson.dump(new, f)


def zarr_init_group_and_store(store=None, zarr_version=None):
zarr_version = zarr_version or 2
if _ZARR_VERSION == 3 and zarr_version == 2:
Expand All @@ -147,6 +152,7 @@ def zarr_init_group_and_store(store=None, zarr_version=None):
store = store or {}
return zarr.group(store, overwrite=True, zarr_version=zarr_version), store


def zarr_open(store, zarr_version=None):
if _ZARR_VERSION == 3:
store = store or StorePath(MemoryStore(mode="w"))
Expand All @@ -155,6 +161,7 @@ def zarr_open(store, zarr_version=None):
store = store or {}
return zarr.open(store, zarr_version=zarr_version)


def _encode_for_JSON(store, zarr_version=2):
"""Make store JSON encodable"""
if _ZARR_VERSION == 2 or zarr_version == 2:
Expand Down

0 comments on commit 54650ac

Please sign in to comment.