get paths from config

cacholote/clean.py

@@ -36,6 +36,10 @@
 )
 
 
+class CleanerError(Exception):
+    pass
+
+
 def _get_files_from_cache_entry(cache_entry: database.CacheEntry) -> dict[str, str]:
     result = cache_entry.result
     if not isinstance(result, (list, tuple, set)):
@@ -93,7 +97,11 @@ def delete(func_to_del: str | Callable[..., Any], *args: Any, **kwargs: Any) ->
 
 
 class _Cleaner:
-    def __init__(self, fs: fsspec.AbstractFileSystem, dirname: str) -> None:
+    def __init__(
+        self,
+        fs: fsspec.AbstractFileSystem,
+        dirname: str,
+    ) -> None:
         self.logger = config.get().logger
         self.fs = fs
         self.dirname = dirname
@@ -281,7 +289,7 @@ def delete_cache_files(
         self.log_disk_usage()
 
         if not self.stop_cleaning(maxsize):
-            raise ValueError(
+            raise CleanerError(
                 (
                     f"Unable to clean {self.dirname!r}."
                     f" Final disk usage: {self.disk_usage!r}."
@@ -320,18 +328,25 @@ def clean_cache_files(
         tags_to_clean and tags_to_keep are mutually exclusive.
     """
     fs, dirnames = utils.get_cache_files_fs_dirnames()
+    exceptions = []
     for dirname in dirnames:
         cleaner = _Cleaner(fs=fs, dirname=dirname)
 
         if delete_unknown_files:
             cleaner.delete_unknown_files(lock_validity_period, recursive)
 
-        cleaner.delete_cache_files(
-            maxsize=maxsize,
-            method=method,
-            tags_to_clean=tags_to_clean,
-            tags_to_keep=tags_to_keep,
-        )
+        try:
+            cleaner.delete_cache_files(
+                maxsize=maxsize,
+                method=method,
+                tags_to_clean=tags_to_clean,
+                tags_to_keep=tags_to_keep,
+            )
+        except CleanerError as exc:
+            exceptions.append(exc)
+
+    if exceptions:
+        raise CleanerError("\n".join(map(str, exceptions)))
 
 
 def clean_invalid_cache_entries(

cacholote/config.py

@@ -18,6 +18,7 @@
 import abc
 import datetime
 import logging
+import os
 import pathlib
 import tempfile
 from types import TracebackType
@@ -42,6 +43,13 @@
 )
 
 
+def _default_cache_files_urlpaths() -> list[str]:
+    config = os.environ.get("CACHOLOTE_CACHE_FILES_URLPATHS_CONFIG")
+    if config:
+        return pathlib.Path(config).read_text().splitlines()
+    return [_DEFAULT_CACHE_FILES_URLPATH]
+
+
 class Context(abc.ABC):
     @abc.abstractmethod
     def __init__(self, *args: Any, **kwargs: Any) -> None: ...
@@ -55,7 +63,9 @@ class Settings(pydantic_settings.BaseSettings):
     cache_db_urlpath: Optional[str] = _DEFAULT_CACHE_DB_URLPATH
     create_engine_kwargs: dict[str, Any] = {}
     sessionmaker: Optional[sa.orm.sessionmaker[sa.orm.Session]] = None
-    cache_files_urlpaths: list[str] = [_DEFAULT_CACHE_FILES_URLPATH]
+    cache_files_urlpaths: list[str] = pydantic.Field(
+        default_factory=_default_cache_files_urlpaths
+    )
     cache_files_urlpath_readonly: Optional[str] = None
     cache_files_storage_options: dict[str, Any] = {}
     xarray_cache_type: Literal[

cacholote/utils.py

@@ -38,7 +38,7 @@ def hexdigestify(text: str) -> str:
 
 
 def get_cache_files_fs_dirnames() -> tuple[fsspec.AbstractFileSystem, list[str]]:
-    """Return the ``fsspec`` filesystem and directory name where cache files are stored."""
+    """Return the ``fsspec`` filesystem and directory names where cache files are stored."""
     paths = []
     for cache_files_urlpath in config.get().cache_files_urlpaths:
         fs, _, (path,) = fsspec.get_fs_token_paths(

tests/test_01_settings.py

@@ -2,7 +2,6 @@
 
 import contextlib
 import datetime
-import os
 import pathlib
 from typing import Any
 
@@ -89,25 +88,28 @@ def test_set_engine_and_sessionmaker(
         assert config.get().instantiated_sessionmaker is old_sessionmaker
 
 
-def test_env_variables(tmp_path: pathlib.Path) -> None:
-    # env variables
-    old_environ = dict(os.environ)
-    os.environ["CACHOLOTE_CACHE_DB_URLPATH"] = "sqlite://"
-
+def test_env_variables(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
     # env file
     dotenv_path = tmp_path / ".env.cacholote"
     with dotenv_path.open("w") as f:
         f.write("CACHOLOTE_IO_DELETE_ORIGINAL=TRUE")
 
+    # config file
+    config_path = tmp_path / "config"
+    cache_files_urlpaths = [str(tmp_path / "foo"), str(tmp_path / "bar")]
+    config_path.write_text("\n".join(cache_files_urlpaths))
+
+    # env variables
+    monkeypatch.setenv("CACHOLOTE_CACHE_DB_URLPATH", "sqlite://")
+    monkeypatch.setenv("CACHOLOTE_CACHE_FILES_URLPATHS_CONFIG", str(config_path))
+
     config.reset(str(dotenv_path))
-    try:
-        assert config.get().cache_db_urlpath == "sqlite://"
-        assert str(config.get().engine.url) == "sqlite://"
-        assert config.get().io_delete_original is True
-        assert str(config.get().engine.url) == "sqlite://"
-    finally:
-        os.environ.clear()
-        os.environ.update(old_environ)
+    settings = config.get()
+    assert settings.cache_db_urlpath == "sqlite://"
+    assert str(settings.engine.url) == "sqlite://"
+    assert settings.io_delete_original is True
+    assert str(settings.engine.url) == "sqlite://"
+    assert settings.cache_files_urlpaths == cache_files_urlpaths
 
 
 @pytest.mark.parametrize("poolclass", ("NullPool", sa.pool.NullPool))