Merge pull request #50 from openweathermap/dev

Dev

deker/client.py

@@ -192,6 +192,7 @@ def __init__(
         write_lock_check_interval: int = 1,
         loglevel: str = "ERROR",
         memory_limit: Union[int, str] = 0,
+        skip_collection_create_memory_check: bool = False,
         **kwargs: Any,
     ) -> None:
         """Deker client constructor.
@@ -209,7 +210,7 @@ def __init__(
           converted into bytes. If result is ``<= 0`` - total RAM + total swap is used
 
           .. note:: This parameter is used for early runtime break in case of potential memory overflow
-
+        :param skip_collection_create_memory_check: If we don't want to check size during collection creation
         :param kwargs: a wildcard, reserved for any extra parameters
         """
         try:
@@ -230,6 +231,7 @@ def __init__(
                 write_lock_check_interval=write_lock_check_interval,
                 loglevel=loglevel.upper(),
                 memory_limit=mem_limit,
+                skip_collection_create_memory_check=skip_collection_create_memory_check,
             )
             self.__uri: Uri = Uri.create(self.__config.uri)
             self.__is_closed: bool = True

deker/collection.py

@@ -37,7 +37,7 @@
     TimeDimensionSchema,
     VArraySchema,
 )
-from deker.tools import check_memory, not_deleted
+from deker.tools import not_deleted
 from deker.types import Serializer
 
 
@@ -278,9 +278,6 @@ def create(
         :param custom_attributes: ``Array`` or ``VArray`` custom attributes
         :param id_: ``Array`` or ``VArray`` unique UUID string
         """
-        schema = self.array_schema
-        shape = schema.arrays_shape if hasattr(schema, "arrays_shape") else schema.shape
-        check_memory(shape, schema.dtype, self.__adapter.ctx.config.memory_limit)
         array = self.__manager.create(primary_attributes, custom_attributes, id_)
         self.logger.debug(
             f"{array.__class__.__name__} id={array.id} {primary_attributes=}, {custom_attributes=} created"

deker/config.py

@@ -43,6 +43,7 @@ class DekerConfig:
     varray_data_directory: str = "varray_data"
     array_symlinks_directory: str = "array_symlinks"
     varray_symlinks_directory: str = "varray_symlinks"
+    skip_collection_create_memory_check: bool = False
 
     @property
     def as_dict(self) -> dict:

docs/deker/fine_tuning.rst

@@ -93,9 +93,12 @@ If you need to change it on the fly, you may use the following function::
    set_logging_level("INFO")  # now DEKER™ logs starting from "INFO" level
 
 
+
 ``memory_limit``
 ----------------
 
+.. _memory_limit:
+
 This parameter is used for the early run time break in case of potential memory overflow.
 
 DEKER™ operates big amounts of data, and you may be unaware that your machine will probably run out
@@ -142,6 +145,21 @@ suffix is ignored: ``"1024k"``, ``"512m"``, ``"8g"``, ``"1t"`` will work.
    If you set a memory limit to your container, you'd better limit DEKER™ to the same value.
    Otherwise your container may be killed because of memory overflow.
 
+``skip_collection_create_memory_check``
+---------------------------------------
+Currently deker has 3 places, where memory check, described in `memory_limit`_:
+
+ * On collection creation via ``client.create_collection()``
+ * On getting subset e.g ``array[:]``
+ * On reading array as ``XArray`` e.g ``array[:].read_xarray()``
+
+While the last two prevent memory overflow and are required,
+sometimes you may need to be able to skip the first one
+
+You can do so by providing
+``skip_collection_create_memory_check=True`` as argument to
+the :meth:`Client <deker.client.Client>` constructor
+
 HDF5 Options
 ============