Skip to content

Commit

Permalink
Rebasing Kvikio checks
Browse files Browse the repository at this point in the history
Signed-off-by: Julio Faracco <[email protected]>
  • Loading branch information
jcfaracco committed Oct 1, 2023
1 parent 8277491 commit c7dc72f
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 3 deletions.
9 changes: 6 additions & 3 deletions dasf/datasets/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,15 @@

from kvikio.nvcomp_codec import NvCompBatchCodec
from kvikio.zarr import GDSStore
USE_KVIKIO=True
except ImportError: # pragma: no cover
USE_KVIKIO=False
pass

from pathlib import Path

from dasf.utils.funcs import human_readable_size
from dasf.utils.funcs import is_kvikio_supported
from dasf.utils.funcs import is_gds_supported
from dasf.utils.funcs import is_nvcomp_codec_supported
from dasf.utils.decorators import task_handler
from dasf.utils.types import is_array
from dasf.utils.types import is_dask_array
Expand Down Expand Up @@ -493,7 +495,8 @@ def _lazy_load(self, xp, **kwargs):
The data (or a Future load object, for `_lazy` operations).
"""
if self._backend == "kvikio" and USE_KVIKIO:
if (self._backend == "kvikio" and is_kvikio_supported() and
is_gds_supported() and is_nvcomp_codec_supported()):
store = GDSStore(self._root_file)
meta = json.loads(store[".zarray"])
meta["compressor"] = NvCompBatchCodec("lz4").get_config()
Expand Down
37 changes: 37 additions & 0 deletions dasf/utils/funcs.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,18 @@
except ImportError: # pragma: no cover
JAX_SUPPORTED = False

try:
import kvikio
KVIKIO_SUPPORTED = True
except ImportError: # pragma: no cover
KVIKIO_SUPPORTED = False

try:
from kvikio.nvcomp_codec import NvCompBatchCodec
NV_COMP_BATCH_CODEC_SUPPORTED = True
except ImportError: # pragma: no cover
NV_COMP_BATCH_CODEC_SUPPORTED = False


def human_readable_size(size, decimal=3) -> str:
"""
Expand Down Expand Up @@ -298,6 +310,31 @@ def is_gpu_supported() -> bool:
return GPU_SUPPORTED and get_gpu_count() >= 1


def is_kvikio_supported() -> bool:
"""
Return if kvikio is supported (installed).
"""
return KVIKIO_SUPPORTED


def is_gds_supported() -> bool:
"""
Return if GPU Direct Store is supported.
"""
if is_kvikio_supported():
props = kvikio.DriverProperties()
return props.is_gds_available

return False


def is_nvcomp_codec_supported() -> bool:
"""
Return if NVidia Compressor Codecs are supported.
"""
return NV_COMP_BATCH_CODEC_SUPPORTED


def is_jax_supported() -> bool:
"""
Return if JAX is supported.
Expand Down

0 comments on commit c7dc72f

Please sign in to comment.