From f58aa00adaf23b4c2d013e7b908544b7db4ebb13 Mon Sep 17 00:00:00 2001 From: Max Hipperson <40496579+maxhipperson@users.noreply.github.com> Date: Tue, 20 Feb 2024 12:39:38 +0000 Subject: [PATCH 1/5] feat: add file extension in functional reader --- uhura/caches.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/uhura/caches.py b/uhura/caches.py index cec646a..c96115c 100644 --- a/uhura/caches.py +++ b/uhura/caches.py @@ -7,7 +7,7 @@ from typing import Callable, DefaultDict, Optional, Protocol, Type # ParamSpec from uhura.composition import async_unit, compose -from uhura.serde import Serde, DEFAULT_SERDE +from uhura.serde import DEFAULT_SERDE, Serde logger = logging.getLogger("uhura.caches") @@ -57,9 +57,18 @@ def update(self, obj): def get(self): return self._serde.read_from_file(self._path) + @classmethod + def render_path(cls, base_path: str, cache_key: str, serde: Serde): + if not cache_key.endswith(serde.file_extension): + cache_key = cache_key + serde.file_extension + return os.path.join(base_path, cache_key) + @classmethod def cache_for_instance(cls, cacheable: Cacheable, base_path: str): - return cls(os.path.join(base_path, cacheable.cache_key()), cacheable.get_serde()) + cache_key = cacheable.cache_key() + serde = cacheable.get_serde() + path = cls.render_path(base_path, cache_key, serde) + return cls(path, serde) async def _take(async_iterable, n=5): From 3d1dc4037ee16d12b81668ee02cc65a2b9a57bc0 Mon Sep 17 00:00:00 2001 From: Max Hipperson <40496579+maxhipperson@users.noreply.github.com> Date: Fri, 23 Feb 2024 11:18:25 +0000 Subject: [PATCH 2/5] feat: serde subclassses need a file extension --- uhura/serde.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/uhura/serde.py b/uhura/serde.py index 35853de..2c82d38 100644 --- a/uhura/serde.py +++ b/uhura/serde.py @@ -1,5 +1,5 @@ import pickle -from typing import TypeVar, Generic, ClassVar, Any +from typing import Any, ClassVar, Generic, TypeVar import pandas as pd @@ -9,6 +9,11 @@ class Serde(Generic[SerdeType]): file_extension: ClassVar[str] + def __init_subclass__(cls) -> None: + assert cls.file_extension is not None, "Serde implementations must have a valid file_extension" + return super().__init_subclass__() + + def read_from_file(self, file) -> SerdeType: raise NotImplementedError() From 96cf84985689e3e4d480c758b74ff7cf8fb15d99 Mon Sep 17 00:00:00 2001 From: Max Hipperson <40496579+maxhipperson@users.noreply.github.com> Date: Fri, 23 Feb 2024 11:19:06 +0000 Subject: [PATCH 3/5] test: should probably test this stuff --- tests/test_caches.py | 14 ++++++++++++++ tests/test_serde.py | 20 +++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/tests/test_caches.py b/tests/test_caches.py index 57297fe..2d2b4d6 100644 --- a/tests/test_caches.py +++ b/tests/test_caches.py @@ -11,6 +11,7 @@ from uhura.base import Readable, Writable from uhura.caches import ( CachingService, + LocalCache, ) from uhura.caching import ( cache_local_input, @@ -20,6 +21,7 @@ ) from uhura.functional import uhura_reader, uhura_writer from uhura.modes import fixture_builder_mode, task_test_mode +from uhura.serde import PickleSerde class RandomReadable(Readable[float]): @@ -293,3 +295,15 @@ def test_cache_key(): assert ( DatabaseReader("id1").read() != DatabaseReader("id2").read() ), "Cache key being ignored" + + +def test_local_cache_path_renders_path_with_file_extension_from_serde_if_not_present(): + cache_path = "my/cache/dir" + serde = PickleSerde() + local_cache = LocalCache(cache_path, serde) + + base_path = "base/path" + cache_key = "filename" + rendered_path = local_cache.render_path(base_path, cache_key, serde) + + assert rendered_path.split(".")[-1] == "pkl" diff --git a/tests/test_serde.py b/tests/test_serde.py index 4e1326e..170f882 100644 --- a/tests/test_serde.py +++ b/tests/test_serde.py @@ -1,8 +1,12 @@ -from uhura.serde import PickleSerde +import os +from tempfile import TemporaryDirectory +from typing import Any + +import pytest + from uhura.base import Readable from uhura.modes import fixture_builder_mode, task_test_mode -from tempfile import TemporaryDirectory -import os +from uhura.serde import PickleSerde, Serde class FakeSerde(PickleSerde): @@ -44,3 +48,13 @@ def get_serde(self): assert not test_serde.has_been_read client.read() assert test_serde.has_been_read + + +def test_serde_subclass_requires_file_extension(): + with pytest.raises(AttributeError): + class Bad(Serde[Any]): + def read_from_file(self, file): + pass + + def write_to_file(self, file, obj: Any): + pass From 24f4f2b321b9b26e22c94553c5c82b0987058972 Mon Sep 17 00:00:00 2001 From: Max Hipperson <40496579+maxhipperson@users.noreply.github.com> Date: Fri, 23 Feb 2024 11:29:31 +0000 Subject: [PATCH 4/5] style: black --- uhura/serde.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uhura/serde.py b/uhura/serde.py index 2c82d38..1e5a6aa 100644 --- a/uhura/serde.py +++ b/uhura/serde.py @@ -10,10 +10,11 @@ class Serde(Generic[SerdeType]): file_extension: ClassVar[str] def __init_subclass__(cls) -> None: - assert cls.file_extension is not None, "Serde implementations must have a valid file_extension" + assert ( + cls.file_extension is not None + ), "Serde implementations must have a valid file_extension" return super().__init_subclass__() - def read_from_file(self, file) -> SerdeType: raise NotImplementedError() From bd0945a8551f845a4b2fd6bf52389ba99e1faa0f Mon Sep 17 00:00:00 2001 From: Max Hipperson <40496579+maxhipperson@users.noreply.github.com> Date: Fri, 23 Feb 2024 11:57:51 +0000 Subject: [PATCH 5/5] test: fix test --- tests/test_serde.py | 3 ++- uhura/serde.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_serde.py b/tests/test_serde.py index 170f882..88ecdb1 100644 --- a/tests/test_serde.py +++ b/tests/test_serde.py @@ -51,7 +51,8 @@ def get_serde(self): def test_serde_subclass_requires_file_extension(): - with pytest.raises(AttributeError): + with pytest.raises(AssertionError): + class Bad(Serde[Any]): def read_from_file(self, file): pass diff --git a/uhura/serde.py b/uhura/serde.py index 1e5a6aa..bee00fc 100644 --- a/uhura/serde.py +++ b/uhura/serde.py @@ -10,8 +10,8 @@ class Serde(Generic[SerdeType]): file_extension: ClassVar[str] def __init_subclass__(cls) -> None: - assert ( - cls.file_extension is not None + assert hasattr( + cls, "file_extension" ), "Serde implementations must have a valid file_extension" return super().__init_subclass__()