Merge pull request #48 from openweathermap/dev

Release v1.1.6

.github/workflows/on_release.yml

@@ -38,7 +38,7 @@ jobs:
       url: https://pypi.org/p/${{ vars.PACKAGE_NAME }}
 
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           # unpacks default artifact into dist/
           # if `name: artifact` is omitted, the action will create extra parent dir

.github/workflows/on_test_release.yml

@@ -21,7 +21,7 @@ jobs:
       url: https://test.pypi.org/p/${{ vars.PACKAGE_NAME }}
 
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           # unpacks default artifact into dist/
           # if `name: artifact` is omitted, the action will create extra parent dir

deker/ABC/base_array.py

@@ -19,7 +19,6 @@
 import json
 
 from abc import ABC, abstractmethod
-from collections import OrderedDict
 from copy import deepcopy
 from datetime import datetime, timedelta
 from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union
@@ -35,10 +34,11 @@
 from deker.schemas import ArraySchema, VArraySchema
 from deker.subset import Subset, VSubset
 from deker.tools.array import check_memory, get_id
+from deker.tools.attributes import make_ordered_dict, serialize_attribute_value
 from deker.tools.schema import create_dimensions
 from deker.types.private.classes import ArrayMeta, Serializer
 from deker.types.private.typings import FancySlice, Numeric, Slice
-from deker.validators import process_attributes, is_valid_uuid, validate_custom_attributes_update
+from deker.validators import is_valid_uuid, process_attributes, validate_custom_attributes_update
 
 
 if TYPE_CHECKING:
@@ -295,10 +295,9 @@ def __init__(
             self.schema, primary_attributes, custom_attributes
         )
 
-        self.primary_attributes: OrderedDict = (
-            OrderedDict({**primary_attributes}) if primary_attributes else OrderedDict()
+        self.primary_attributes, self.custom_attributes = make_ordered_dict(
+            primary_attributes, custom_attributes, self.schema.attributes  # type: ignore[arg-type]
         )
-        self.custom_attributes: dict = custom_attributes if custom_attributes else {}
 
     def __del__(self) -> None:
         del self.__adapter
@@ -392,26 +391,14 @@ def update_custom_attributes(self, attributes: dict) -> None:
         self.logger.info(f"{self!s} custom attributes updated: {attributes}")
 
     def _create_meta(self) -> str:
-        """Serialize array into metadata string."""
+        """Serialize array into metadata JSON string."""
         primary_attrs, custom_attrs = deepcopy(self.primary_attributes), deepcopy(
             self.custom_attributes
         )
         for attrs in (primary_attrs, custom_attrs):
             for key, value in attrs.items():
-                if isinstance(value, datetime):
-                    attrs[key] = value.isoformat()
-                elif isinstance(value, np.ndarray):
-                    attrs[key] = value.tolist()
-                elif isinstance(value, (list, tuple)):
-                    elements = []
-                    for element in value:
-                        if isinstance(element, np.integer):
-                            elements.append(int(element))
-                        else:
-                            elements.append(element)
-                    attrs[key] = tuple(elements)
-                else:
-                    attrs[key] = value
+                attrs[key] = serialize_attribute_value(value)
+
         return json.dumps(
             {
                 "id": self.id,
@@ -470,34 +457,21 @@ def _create_from_meta(
             attrs_schema = collection.varray_schema.attributes
         else:
             attrs_schema = collection.array_schema.attributes
-        try:
-            for attr in attrs_schema:
-                attributes = (
-                    meta["primary_attributes"] if attr.primary else meta["custom_attributes"]
-                )
 
-                value = attributes[attr.name]
-
-                if attr.dtype == datetime:
-                    attributes[attr.name] = get_utc(value)
-                if attr.dtype == tuple:
-                    if (
-                        attr.primary or (not attr.primary and value is not None)
-                    ) and not isinstance(value, list):
-                        raise DekerMetaDataError(
-                            f"Collection '{collection.name}' metadata is corrupted: "
-                            f"attribute '{attr.name}' has invalid type '{type(value)}'; '{attr.dtype}' expected"
-                        )
-
-                    if attr.primary or (not attr.primary and value is not None):
-                        attributes[attr.name] = tuple(value)
+        try:
+            # To ensure the order of attributes
+            primary_attributes, custom_attributes = make_ordered_dict(
+                meta["primary_attributes"],
+                meta["custom_attributes"],
+                attrs_schema,  # type: ignore[arg-type]
+            )
 
             arr_params = {
                 "collection": collection,
                 "adapter": array_adapter,
                 "id_": meta["id"],
-                "primary_attributes": meta.get("primary_attributes"),
-                "custom_attributes": meta.get("custom_attributes"),
+                "primary_attributes": primary_attributes,
+                "custom_attributes": custom_attributes,
             }
             if varray_adapter:
                 arr_params.update({"adapter": varray_adapter, "array_adapter": array_adapter})

deker/ABC/base_locks.py

@@ -15,7 +15,6 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 """Abstract interfaces for locks."""
-
 from abc import ABC, abstractmethod
 from functools import wraps
 from pathlib import Path
@@ -31,16 +30,15 @@ class BaseLock(SelfLoggerMixin, ABC):
     ALLOWED_TYPES: List[str] = []
     lock: Optional[Union[Flock, Path]] = None
     instance: Optional[Any] = None
+    args: Optional[List[Any]] = None
+    kwargs: Optional[Dict] = None
 
     @abstractmethod
-    def get_path(self, func_args: Sequence, func_kwargs: Dict) -> Optional[Path]:
+    def get_path(self) -> Optional[Path]:
         """Get path to the lock file.
 
         For Arrays it shall be .arrlock (array read lock) or path to the file (array write lock)
         For VArrays there are no specific locks for reading, for writing - lock on .json
-
-        :param func_args: Arguments of method call
-        :param func_kwargs: Keyword arguments of method call
         """
         pass
 
@@ -98,7 +96,7 @@ def _inner_method_logic(lock: "BaseLock", args: Sequence, kwargs: Dict, func: Ca
         :param kwargs: Keyword arguments of decorated function
         """
         lock.check_existing_lock(args, kwargs)
-        path = lock.get_path(args, kwargs)
+        path = lock.get_path()
         lock.acquire(path)
         try:
             # Logic is in the separate method, so we can override its behavior
@@ -129,7 +127,8 @@ def inner(*args: Sequence, **kwargs: Dict[str, Any]) -> Any:
             lock = self.__class__()
             # as we wrap methods, we should have access to 'self' objects
             lock.instance = kwargs.get("self") or args[0]
-
+            lock.args = args
+            lock.kwargs = kwargs
             # Check that we don't have lock on anything that besides methods that require lock
             lock.check_type()
 

deker/__init__.py

@@ -17,11 +17,6 @@
 # nopycln: file
 # isort: skip_file
 
-from deker_local_adapters.storage_adapters.hdf5.hdf5_options import (
-    HDF5Options,
-    HDF5CompressionOpts,
-)
-
 from deker.arrays import Array, VArray
 from deker.client import Client
 from deker.collection import Collection
@@ -36,6 +31,10 @@
 )
 from deker.subset import Subset, VSubset
 from deker.types.public.classes import Scale
+from deker_local_adapters.storage_adapters.hdf5.hdf5_options import (
+    HDF5Options,
+    HDF5CompressionOpts,
+)
 
 __all__ = (
     # deker.adapters.hdf5

deker/client.py

@@ -27,8 +27,8 @@
 from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Tuple, Type, Union
 
 from deker_tools.data import convert_size_to_human
-from deker_tools.path import is_path_valid
 from deker_tools.log import set_logger
+from deker_tools.path import is_path_valid
 from psutil import swap_memory, virtual_memory
 from tqdm import tqdm
 
@@ -43,7 +43,7 @@
 )
 from deker.integrity import IntegrityChecker
 from deker.locks import META_DIVIDER
-from deker.log import SelfLoggerMixin, set_logging_level, format_string
+from deker.log import SelfLoggerMixin, format_string, set_logging_level
 from deker.schemas import ArraySchema, VArraySchema
 from deker.tools import convert_human_memory_to_bytes
 from deker.types import ArrayLockMeta, CollectionLockMeta, LocksExtensions, LocksTypes, StorageSize

deker/errors.py

@@ -13,6 +13,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List
 
 
 class DekerBaseApplicationError(Exception):
@@ -106,9 +107,44 @@ class DekerSubsetError(DekerArrayError):
     """If something goes wrong while Subset managing."""
 
 
-class DekerVSubsetError(DekerSubsetError):
-    """If something goes wrong while VSubset managing."""
+class DekerExceptionGroup(DekerBaseApplicationError):
+    """If one or more threads finished with any exception.
+
+    Name, message and reasonable tracebacks of all
+    of these exceptions shall be collected in a list
+    and passed to this class along with some message.
+
+    ```
+    futures = [executor.submit(func, arg) for arg in iterable]
+
+    exceptions = []
+    for future in futures:
+        try:
+            future.result()
+        except Exception as e:
+            exceptions.append(repr(e) + "\n" + traceback.format_exc(-1))
+    ```
+    """
+
+    def __init__(self, message: str, exceptions: List[str]):
+        self.message = message
+        self.exceptions = exceptions
+        super().__init__(message)
+
+    def __str__(self) -> str:
+        enumerated = [f"{num}) {e}" for num, e in enumerate(self.exceptions, start=1)]
+        joined = "\n".join(str(e) for e in enumerated)
+        return f"{self.message}; exceptions:\n\n{joined} "
 
 
 class DekerMemoryError(DekerBaseApplicationError, MemoryError):
     """Early memory overflow exception."""
+
+
+class DekerVSubsetError(DekerSubsetError, DekerExceptionGroup):
+    """If something goes wrong while VSubset managing.
+
+    Regarding that VSubset's inner Subsets' processing
+    is made in an Executor, this exception actually is
+    an `exceptions messages group`.
+    """

deker/integrity.py

@@ -33,6 +33,7 @@
 from deker.tools import get_main_path, get_symlink_path
 from deker.types.private.enums import LocksExtensions
 
+
 if TYPE_CHECKING:
     from deker.client import Client
 
@@ -187,15 +188,15 @@ def _check_varrays_or_arrays(
                 except DekerBaseApplicationError as e:
                     if self.stop_on_error:
                         raise DekerIntegrityError(str(e))
-                    self.errors[
-                        f"Collection {collection.name} arrays integrity errors:"
-                    ].append(str(e))
+                    self.errors[f"Collection {collection.name} arrays integrity errors:"].append(
+                        str(e)
+                    )
         except DekerMetaDataError as e:
             if self.stop_on_error:
                 raise e
-            self.errors[
-                f"Collection {collection.name} (V)Arrays initialization errors:"
-            ].append(str(e))
+            self.errors[f"Collection {collection.name} (V)Arrays initialization errors:"].append(
+                str(e)
+            )
 
     def check(self, collection: Collection) -> None:
         """Check if Arrays or VArrays and their locks in Collection are valid.