From 13d6615b89c0b36236f39ef34b0127a778d289cf Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Mon, 8 Jan 2024 10:21:18 +0100 Subject: [PATCH] 24.01.0 (#432) --- .github/workflows/publish-dockerhub.yml | 2 + .github/workflows/publish-pypi.yml | 10 +- docs/installation.rst | 14 +- docs/logging.rst | 17 +- docs/requirements.txt | 6 +- docs/util.rst | 134 ++++++++++ readme.md | 10 +- requirements.txt | 4 +- requirements_setup.txt | 12 +- requirements_tests.txt | 4 +- run/conf/logging.yml | 8 +- .../rules/openhab/test_persistence.py | 41 ++- src/HABApp/__version__.py | 4 +- src/HABApp/config/logging/__init__.py | 2 +- src/HABApp/config/logging/default_logfile.py | 2 + src/HABApp/config/logging/handler.py | 26 ++ src/HABApp/core/const/hints.py | 8 + src/HABApp/core/errors.py | 22 ++ src/HABApp/core/types/color.py | 70 +++-- src/HABApp/mqtt/connection/handler.py | 3 + src/HABApp/openhab/connection/connection.py | 2 + .../openhab/connection/handler/handler.py | 4 +- .../openhab/connection/handler/helper.py | 7 +- src/HABApp/openhab/connection/plugins/out.py | 4 +- .../definitions/helpers/persistence_data.py | 1 + src/HABApp/openhab/items/base_item.py | 5 +- src/HABApp/openhab/items/commands.py | 29 +- src/HABApp/openhab/items/contact_item.py | 6 +- src/HABApp/openhab/items/dimmer_item.py | 29 +- src/HABApp/openhab/items/number_item.py | 14 +- .../openhab/items/rollershutter_item.py | 20 +- src/HABApp/openhab/items/switch_item.py | 35 ++- src/HABApp/openhab/map_items.py | 13 +- src/HABApp/util/__init__.py | 1 + src/HABApp/util/functions/min_max.py | 10 +- src/HABApp/util/rate_limiter/__init__.py | 1 + src/HABApp/util/rate_limiter/limiter.py | 158 +++++++++++ .../util/rate_limiter/limits/__init__.py | 3 + src/HABApp/util/rate_limiter/limits/base.py | 74 ++++++ .../util/rate_limiter/limits/fixed_window.py | 51 ++++ .../util/rate_limiter/limits/leaky_bucket.py | 50 ++++ src/HABApp/util/rate_limiter/parser.py | 27 ++ src/HABApp/util/rate_limiter/registry.py | 26 ++ tests/test_core/test_types/test_color.py | 18 +- .../test_openhab/test_items/test_commands.py | 12 +- tests/test_openhab/test_items/test_contact.py | 10 + tests/test_openhab/test_items/test_dimmer.py | 22 ++ tests/test_openhab/test_items/test_number.py | 19 ++ .../test_items/test_rollershutter.py | 14 + tests/test_openhab/test_items/test_switch.py | 21 ++ .../test_plugins/test_load_items.py | 4 +- tests/test_utils/test_rate_limiter.py | 251 ++++++++++++++++++ 52 files changed, 1220 insertions(+), 120 deletions(-) create mode 100644 src/HABApp/util/rate_limiter/__init__.py create mode 100644 src/HABApp/util/rate_limiter/limiter.py create mode 100644 src/HABApp/util/rate_limiter/limits/__init__.py create mode 100644 src/HABApp/util/rate_limiter/limits/base.py create mode 100644 src/HABApp/util/rate_limiter/limits/fixed_window.py create mode 100644 src/HABApp/util/rate_limiter/limits/leaky_bucket.py create mode 100644 src/HABApp/util/rate_limiter/parser.py create mode 100644 src/HABApp/util/rate_limiter/registry.py create mode 100644 tests/test_openhab/test_items/test_dimmer.py create mode 100644 tests/test_openhab/test_items/test_rollershutter.py create mode 100644 tests/test_openhab/test_items/test_switch.py create mode 100644 tests/test_utils/test_rate_limiter.py diff --git a/.github/workflows/publish-dockerhub.yml b/.github/workflows/publish-dockerhub.yml index 1ea4c7db..ff3c08a6 100644 --- a/.github/workflows/publish-dockerhub.yml +++ b/.github/workflows/publish-dockerhub.yml @@ -9,6 +9,8 @@ on: jobs: buildx: runs-on: ubuntu-latest + environment: release + steps: - uses: actions/checkout@v2 with: diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 15a12ece..2aac5900 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -7,11 +7,16 @@ jobs: build-n-publish: name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI runs-on: ubuntu-latest + environment: release + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write steps: - uses: actions/checkout@v3 with: ref: master + - name: Set up Python 3.10 uses: actions/setup-python@v4 with: @@ -27,7 +32,4 @@ jobs: python setup.py sdist bdist_wheel - name: Publish distribution to PyPI - uses: pypa/gh-action-pypi-publish@master - with: - user: __token__ - password: ${{ secrets.pypi_api_key }} + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/docs/installation.rst b/docs/installation.rst index 1b8a30d6..7b85c7bb 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -59,8 +59,8 @@ Installation After the installation take a look how to configure HABApp. A default configuration will be created on the first start. -Upgrading -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Upgrade to the latest version +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #. Stop HABApp #. Activate the virtual environment @@ -87,6 +87,16 @@ Upgrading #. Observe the logs for errors in case there were changes +Installation of a certain version +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Installing a certain version of HABApp requires the same steps used for installation or upgrading HABApp. +However the final ``python3 -m pip install`` command is now different and contains the version number:: + + python3 -m pip install HABApp==23.12.0 + +The complete list of available versions can be found on `pypi `_. + Autostart after reboot ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Check where habapp is installed:: diff --git a/docs/logging.rst b/docs/logging.rst index 297fe85d..55f4af58 100644 --- a/docs/logging.rst +++ b/docs/logging.rst @@ -17,6 +17,17 @@ but the format should be pretty straight forward. | That way even if the HABApp configuration is invalid HABApp can still log the errors that have occurred. | e.g.: ``/HABApp/logs/habapp.log`` or ``c:\HABApp\logs\habapp.log`` +Provided loggers +====================================== + +The ``HABApp.config.logging`` module provides additional loggers which can be used + + +.. autoclass:: HABApp.config.logging.MidnightRotatingFileHandler + +.. autoclass:: HABApp.config.logging.CompressedMidnightRotatingFileHandler + + Example ====================================== @@ -42,7 +53,7 @@ to the file configuration under ``handlers`` in the ``logging.yml``. ... MyRuleHandler: # <-- This is the name of the handler - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.MidnightRotatingFileHandler filename: 'c:\HABApp\Logs\MyRule.log' maxBytes: 10_000_000 backupCount: 3 @@ -84,7 +95,7 @@ Full Example configuration # ----------------------------------------------------------------------------------- handlers: HABApp_default: - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.MidnightRotatingFileHandler filename: 'HABApp.log' maxBytes: 10_000_000 backupCount: 3 @@ -93,7 +104,7 @@ Full Example configuration level: DEBUG MyRuleHandler: - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.MidnightRotatingFileHandler filename: 'c:\HABApp\Logs\MyRule.log' # absolute filename is recommended maxBytes: 10_000_000 backupCount: 3 diff --git a/docs/requirements.txt b/docs/requirements.txt index b3f65219..8032d1f4 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ # Packages required to build the documentation -sphinx == 7.2.5 -sphinx-autodoc-typehints == 1.24.0 -sphinx_rtd_theme == 1.3.0 +sphinx == 7.2.6 +sphinx-autodoc-typehints == 1.25.2 +sphinx_rtd_theme == 2.0.0 sphinx-exec-code == 0.10 autodoc_pydantic == 2.0.1 sphinx-copybutton == 0.5.2 diff --git a/docs/util.rst b/docs/util.rst index 4e5f04ed..1499c06c 100644 --- a/docs/util.rst +++ b/docs/util.rst @@ -75,6 +75,140 @@ Converts a hsb value to the rgb color space .. autofunction:: HABApp.util.functions.hsb_to_rgb +Rate limiter +------------------------------ +A simple rate limiter implementation which can be used in rules. +The limiter is not rule bound so the same limiter can be used in multiples files. +It also works as expected across rule reloads. + + +Defining limits +^^^^^^^^^^^^^^^^^^ +Limits can either be explicitly added or through a textual description. +If the limit does already exist it will not be added again. +It's possible to explicitly create the limits or through some small textual description with the following syntax: + +.. code-block:: text + + [count] [per|in|/] [count (optional)] [s|sec|second|m|min|minute|hour|h|day|month|year] [s (optional)] + +Whitespaces are ignored and can be added as desired + +Examples: + +* ``5 per minute`` +* ``20 in 15 mins`` +* ``300 / hour`` + + +Fixed window elastic expiry algorithm +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This algorithm implements a fixed window with elastic expiry. +That means if the limit is hit the interval time will be increased by the expiry time. + +For example ``3 per minute``: + +* First hit comes ``00:00:00``. Two more hits at ``00:00:59``. + All three pass, intervall goes from ``00:00:00`` - ``00:01:00``. + Another hit comes at ``00:01:01`` an passes. The intervall now goes from ``00:01:01`` - ``00:02:01``. + +* First hit comes ``00:00:00``. Two more hits at ``00:00:30``. All three pass. + Another hit comes at ``00:00:45``, which gets rejected and the intervall now goes from ``00:00:00`` - ``00:01:45``. + A rejected hit makes the interval time longer by expiry time. If another hit comes at ``00:01:30`` it + will also get rejected and the intervall now goes from ``00:00:00`` - ``00:02:30``. + + +Leaky bucket algorithm +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The leaky bucket algorithm is based on the analogy of a bucket that leaks at a constant rate. +As long as the bucket is not full the hits will pass. If the bucket overflows the hits will get rejected. +Since the bucket leaks at a constant rate it will gradually get empty again thus allowing hits to pass again. + + +Example +^^^^^^^^^^^^^^^^^^ + +.. exec_code:: + + from HABApp.util import RateLimiter + + # Create or get existing, name is case insensitive + limiter = RateLimiter('MyRateLimiterName') + + # define limits, duplicate limits of the same algorithm will only be added once + # These lines all define the same limit so it'll result in only one limiter added + limiter.add_limit(5, 60) # add limits explicitly + limiter.parse_limits('5 per minute').parse_limits('5 in 60s', '5/60seconds') # add limits through text + + # add additional limit with leaky bucket algorithm + limiter.add_limit(10, 100, algorithm='leaky_bucket') + + # add additional limit with fixed window elastic expiry algorithm + limiter.add_limit(10, 100, algorithm='fixed_window_elastic_expiry') + + # Test the limit without increasing the hits + for _ in range(100): + assert limiter.test_allow() + + # the limiter will allow 5 calls ... + for _ in range(5): + assert limiter.allow() + + # and reject the 6th + assert not limiter.allow() + + # It's possible to get statistics about the limiter and the corresponding windows + print(limiter.info()) + + # There is a counter that keeps track of the total skips that can be reset + print('Counter:') + print(limiter.total_skips) + limiter.reset() # Can be reset + print(limiter.total_skips) + + +Recommendation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Limiting external requests to an external API works well with the leaky bucket algorithm (maybe with some initial hits). +For limiting notifications the best results can be achieved by combining both algorithms. +Fixed window elastic expiry will notify but block until an issue is resolved, +that's why it's more suited for small intervals. Leaky bucket will allow hits even while the issue persists, +that's why it's more suited for larger intervals. + +.. exec_code:: + + from HABApp.util import RateLimiter + + limiter = RateLimiter('MyNotifications') + limiter.parse_limits('5 in 1 minute', algorithm='fixed_window_elastic_expiry') + limiter.parse_limits("20 in 1 hour", algorithm='leaky_bucket') + + +Documentation +^^^^^^^^^^^^^^^^^^ +.. autofunction:: HABApp.util.RateLimiter + + +.. autoclass:: HABApp.util.rate_limiter.limiter.Limiter + :members: + :inherited-members: + +.. autoclass:: HABApp.util.rate_limiter.limiter.LimiterInfo + :members: + :inherited-members: + +.. autoclass:: HABApp.util.rate_limiter.limiter.FixedWindowElasticExpiryLimitInfo + :members: + :inherited-members: + +.. autoclass:: HABApp.util.rate_limiter.limiter.LeakyBucketLimitInfo + :members: + :inherited-members: + + Statistics ------------------------------ diff --git a/readme.md b/readme.md index 0662d034..ca924f6a 100644 --- a/readme.md +++ b/readme.md @@ -127,7 +127,15 @@ MyOpenhabRule() ``` # Changelog -#### 23.11.1 (2023-11-23) +#### 24.01.0 (2024-01-08) +- Added HABApp.util.RateLimiter +- Added CompressedMidnightRotatingFileHandler +- Updated dependencies +- Small improvement for RGB and HSB types +- Small improvements for openHAB items +- Added toggle for SwitchItem + +#### 23.11.0 (2023-11-23) - Fix for very small float values (#425) - Fix for writing to persistence (#424) - Updated dependencies diff --git a/requirements.txt b/requirements.txt index 9bcc4000..57c6b701 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,8 +6,8 @@ # ----------------------------------------------------------------------------- # Packages for source formatting # ----------------------------------------------------------------------------- -pre-commit >= 3.5, < 3.6 -ruff >= 0.1.6, < 0.2 +pre-commit == 3.5.0 # 3.6.0 requires python >= 3.10 +ruff == 0.1.11 # ----------------------------------------------------------------------------- # Packages for other developement tasks diff --git a/requirements_setup.txt b/requirements_setup.txt index 9251d495..f635f60b 100644 --- a/requirements_setup.txt +++ b/requirements_setup.txt @@ -1,21 +1,21 @@ -aiohttp == 3.9.0 -pydantic == 2.5.2 -msgspec == 0.18.4 -pendulum == 2.1.2 +aiohttp == 3.9.1 +pydantic == 2.5.3 +msgspec == 0.18.5 bidict == 0.22.1 watchdog == 3.0.0 -ujson == 5.8.0 +ujson == 5.9.0 aiomqtt == 1.2.1 immutables == 0.20 eascheduler == 0.1.11 easyconfig == 0.3.1 +pendulum == 2.1.2 stack_data == 0.6.3 colorama == 0.4.6 voluptuous == 0.14.1 -typing-extensions == 4.8.0 +typing-extensions == 4.9.0 aiohttp-sse-client == 0.2.1 diff --git a/requirements_tests.txt b/requirements_tests.txt index 34746f45..9451f3ee 100644 --- a/requirements_tests.txt +++ b/requirements_tests.txt @@ -7,5 +7,5 @@ # Packages to run source tests # ----------------------------------------------------------------------------- packaging == 23.2 -pytest == 7.4.3 -pytest-asyncio == 0.21.1 +pytest == 7.4.4 +pytest-asyncio == 0.23.3 diff --git a/run/conf/logging.yml b/run/conf/logging.yml index 6b8b363a..d4b0ce30 100644 --- a/run/conf/logging.yml +++ b/run/conf/logging.yml @@ -8,13 +8,15 @@ handlers: # There are several Handlers available: # - logging.handlers.RotatingFileHandler: # Will rotate when the file reaches a certain size (see python logging documentation for args) - # - HABApp.core.lib.handler.MidnightRotatingFileHandler: + # - HABApp.config.logging.handler.MidnightRotatingFileHandler: # Will wait until the file reaches a certain size and then will rotate on midnight + # - HABApp.config.logging.handler.CompressedMidnightRotatingFileHandler: + # Same as MidnightRotatingFileHandler but will rotate to a gzipped archive # - More handlers: # https://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler HABApp_default: - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.handler.MidnightRotatingFileHandler filename: 'HABApp.log' maxBytes: 1_048_576 backupCount: 3 @@ -23,7 +25,7 @@ handlers: level: DEBUG EventFile: - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.handler.CompressedMidnightRotatingFileHandler filename: 'events.log' maxBytes: 1_048_576 backupCount: 3 diff --git a/run/conf_testing/rules/openhab/test_persistence.py b/run/conf_testing/rules/openhab/test_persistence.py index 983d204f..e5d5cb0a 100644 --- a/run/conf_testing/rules/openhab/test_persistence.py +++ b/run/conf_testing/rules/openhab/test_persistence.py @@ -1,11 +1,16 @@ from __future__ import annotations from datetime import datetime, timedelta -from typing import Final, Any +from typing import TYPE_CHECKING, Any, Final -from HABApp.openhab.definitions.helpers import OpenhabPersistenceData +from HABAppTests import ItemWaiter, TestBaseRule + +from HABApp.core.connections import Connections from HABApp.openhab.items import NumberItem -from HABAppTests import TestBaseRule + + +if TYPE_CHECKING: + from HABApp.openhab.definitions.helpers import OpenhabPersistenceData class TestPersistenceBase(TestBaseRule): @@ -20,6 +25,9 @@ def __init__(self, service_name: str, item_name: str): def set_up(self): i = NumberItem.get_item(self.item_name) + if i.value is None: + i.oh_post_update(0) + ItemWaiter(self.item_name).wait_for_state(0) i.oh_post_update(int(i.value) + 1 if i.value < 10 else 0) def test_service_available(self): @@ -64,3 +72,30 @@ def test_get(self): TestMapDB() + + +class TestInMemory(TestPersistenceBase): + + def __init__(self): + super().__init__('inmemory', 'RRD4J_Item') + + if Connections.get('openhab').context.version >= (4, 1): + self.add_test('InMemory', self.test_in_memory) + else: + print('Skip "TestInMemory" because of no InMemoryDb') + + def test_in_memory(self): + now = datetime.now().replace(microsecond=0) + t1 = now - timedelta(milliseconds=100) + t2 = now + timedelta(milliseconds=100) + + self.set_persistence_data(t1, 5) + self.set_persistence_data(now, 6) + self.set_persistence_data(t2, 7) + value = self.get_persistence_data(now - timedelta(milliseconds=200), now + timedelta(milliseconds=200)) + + objs = value.get_data() + assert objs == {t1.timestamp(): 5, now.timestamp(): 6, t2.timestamp(): 7} + + +TestInMemory() diff --git a/src/HABApp/__version__.py b/src/HABApp/__version__.py index 3bd0717c..a97afa18 100644 --- a/src/HABApp/__version__.py +++ b/src/HABApp/__version__.py @@ -8,6 +8,6 @@ # - 23.10.0 # # Development versions contain the DEV-COUNTER postfix: -# - 23.09.0.DEV-1 +# - 24.01.0.DEV-1 -__version__ = '23.11.0' +__version__ = '24.01.0' diff --git a/src/HABApp/config/logging/__init__.py b/src/HABApp/config/logging/__init__.py index ac466ba4..b0c0e311 100644 --- a/src/HABApp/config/logging/__init__.py +++ b/src/HABApp/config/logging/__init__.py @@ -1,4 +1,4 @@ -from .handler import MidnightRotatingFileHandler +from .handler import MidnightRotatingFileHandler, CompressedMidnightRotatingFileHandler # isort: split diff --git a/src/HABApp/config/logging/default_logfile.py b/src/HABApp/config/logging/default_logfile.py index 67f58d71..3326232a 100644 --- a/src/HABApp/config/logging/default_logfile.py +++ b/src/HABApp/config/logging/default_logfile.py @@ -18,6 +18,8 @@ def get_default_logfile() -> str: # Will rotate when the file reaches a certain size (see python logging documentation for args) # - HABApp.config.logging.MidnightRotatingFileHandler: # Will wait until the file reaches a certain size and then will rotate on midnight + # - HABApp.config.logging.CompressedMidnightRotatingFileHandler: + # Same as MidnightRotatingFileHandler but will rotate to a gzipped archive # - More handlers: # https://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler diff --git a/src/HABApp/config/logging/handler.py b/src/HABApp/config/logging/handler.py index 95cb4a01..ad920d22 100644 --- a/src/HABApp/config/logging/handler.py +++ b/src/HABApp/config/logging/handler.py @@ -1,8 +1,14 @@ +import gzip +import shutil from datetime import date, datetime from logging.handlers import RotatingFileHandler +from pathlib import Path class MidnightRotatingFileHandler(RotatingFileHandler): + """A rotating file handler that checks once after midnight if the configured size has been exceeded and + then rotates the file + """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -14,3 +20,23 @@ def shouldRollover(self, record): return 0 self.last_check = date return super().shouldRollover(record) + + +class CompressedMidnightRotatingFileHandler(MidnightRotatingFileHandler): + """Same as ``MidnightRotatingFileHandler`` but rotates the file to a gzipped archive (``.gz``) + + """ + + def __init__(self, *args, **kwargs): + self.namer = self.compressed_namer + self.rotator = self.compressed_rotator + super().__init__(*args, **kwargs) + + def compressed_namer(self, default_name: str) -> str: + return default_name + ".gz" + + def compressed_rotator(self, source: str, dest: str): + src = Path(source) + with src.open('rb') as f_in, gzip.open(dest, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + src.unlink() diff --git a/src/HABApp/core/const/hints.py b/src/HABApp/core/const/hints.py index 9fe059e1..ffdf34bb 100644 --- a/src/HABApp/core/const/hints.py +++ b/src/HABApp/core/const/hints.py @@ -1,6 +1,7 @@ from typing import Any as __Any from typing import Awaitable as __Awaitable from typing import Callable as __Callable +from typing import Protocol as __Protocol from typing import Type as __Type from .const import PYTHON_310 as __IS_GE_PYTHON_310 @@ -16,3 +17,10 @@ TYPE_FUNC_ASYNC: TypeAlias = __Callable[..., __Awaitable[__Any]] TYPE_EVENT_CALLBACK: TypeAlias = __Callable[[__Any], __Any] + + +# noinspection PyPropertyDefinition +class HasNameAttr(__Protocol): + @property + def name(self) -> str: + ... diff --git a/src/HABApp/core/errors.py b/src/HABApp/core/errors.py index 59128461..fa87be13 100644 --- a/src/HABApp/core/errors.py +++ b/src/HABApp/core/errors.py @@ -1,3 +1,6 @@ +from HABApp.core.const.hints import HasNameAttr as _HasNameAttr + + class HABAppException(Exception): pass @@ -33,3 +36,22 @@ class ContextBoundObjectIsAlreadyLinkedError(HABAppException): class ContextBoundObjectIsAlreadyUnlinkedError(HABAppException): pass + + +# ---------------------------------------------------------------------------------------------------------------------- +# Value errors +# ---------------------------------------------------------------------------------------------------------------------- +class HABAppValueError(ValueError, HABAppException): + pass + + +class ItemValueIsNoneError(HABAppValueError): + @classmethod + def from_item(cls, item: _HasNameAttr): + return cls(f'Item value is None (item "{item.name:s}")') + + +class InvalidItemValue(HABAppValueError): + @classmethod + def from_item(cls, item: _HasNameAttr, value): + return cls(f'Invalid value for {item.__class__.__name__} {item.name:s}: {value}') diff --git a/src/HABApp/core/types/color.py b/src/HABApp/core/types/color.py index 52d4be76..66dec2e4 100644 --- a/src/HABApp/core/types/color.py +++ b/src/HABApp/core/types/color.py @@ -1,6 +1,6 @@ from colorsys import hsv_to_rgb as _hsv_to_rgb from colorsys import rgb_to_hsv as _rgb_to_hsv -from typing import Tuple, Union, Optional +from typing import Optional, Tuple, Union from typing_extensions import Self @@ -90,19 +90,30 @@ def __str__(self): def __eq__(self, other): if isinstance(other, self.__class__): return self._r == other._r and self._g == other._g and self._b == other._b - elif isinstance(other, HSB): + if isinstance(other, HSB): return self == self.__class__.from_hsb(other) - else: - return NotImplemented - - def __getitem__(self, item: int) -> int: - if item == 0: - return self._r - if item == 1: - return self._g - if item == 2: - return self._b - raise IndexError() + return NotImplemented + + def __getitem__(self, item: Union[int, str]) -> int: + if isinstance(item, int): + if item == 0: + return self._r + if item == 1: + return self._g + if item == 2: + return self._b + raise IndexError() + + if isinstance(item, str): + if item in ('r', 'red'): + return self._r + if item in ('g', 'green'): + return self._g + if item in ('b', 'blue'): + return self._b + raise KeyError() + + raise TypeError() # ------------------------------------------------------------------------------------------------------------------ # Conversions @@ -227,17 +238,28 @@ def __eq__(self, other): return self._hue == other._hue and \ self._saturation == other._saturation and \ self._brightness == other._brightness - else: - return NotImplemented - - def __getitem__(self, item: int) -> float: - if item == 0: - return self._hue - if item == 1: - return self._saturation - if item == 2: - return self._brightness - raise IndexError() + return NotImplemented + + def __getitem__(self, item: Union[int, str]) -> float: + if isinstance(item, int): + if item == 0: + return self._hue + if item == 1: + return self._saturation + if item == 2: + return self._brightness + raise IndexError() + + if isinstance(item, str): + if item in ('h', 'hue'): + return self._hue + if item in ('s', 'saturation'): + return self._saturation + if item in ('b', 'brightness'): + return self._brightness + raise KeyError() + + raise TypeError() # ------------------------------------------------------------------------------------------------------------------ # Conversions diff --git a/src/HABApp/mqtt/connection/handler.py b/src/HABApp/mqtt/connection/handler.py index ebb63b94..4c0be234 100644 --- a/src/HABApp/mqtt/connection/handler.py +++ b/src/HABApp/mqtt/connection/handler.py @@ -72,6 +72,9 @@ async def on_disconnected(self, connection: MqttConnection, context: CONTEXT_TYP assert context is not None connection.log.info('Disconnected') + # remove this check when https://github.com/sbtinstruments/aiomqtt/pull/249 gets merged + if not context._lock.locked(): + await context._lock.acquire() await context.__aexit__(None, None, None) diff --git a/src/HABApp/openhab/connection/connection.py b/src/HABApp/openhab/connection/connection.py index 00a3d825..54bf6ee5 100644 --- a/src/HABApp/openhab/connection/connection.py +++ b/src/HABApp/openhab/connection/connection.py @@ -33,6 +33,8 @@ class OpenhabContext: session: aiohttp.ClientSession session_options: dict[str, Any] + workaround_small_floats: bool + CONTEXT_TYPE: TypeAlias = Optional[OpenhabContext] diff --git a/src/HABApp/openhab/connection/handler/handler.py b/src/HABApp/openhab/connection/handler/handler.py index 2bba83e7..6958547c 100644 --- a/src/HABApp/openhab/connection/handler/handler.py +++ b/src/HABApp/openhab/connection/handler/handler.py @@ -172,7 +172,9 @@ async def on_connecting(self, connection: OpenhabConnection): version=vers, is_oh3=vers < (4, 0), waited_for_openhab=False, created_items={}, created_things={}, - session=self.session, session_options=self.options + session=self.session, session_options=self.options, + + workaround_small_floats=vers < (4, 1) ) # during startup we get OpenhabCredentialsInvalidError even though credentials are correct diff --git a/src/HABApp/openhab/connection/handler/helper.py b/src/HABApp/openhab/connection/handler/helper.py index eccf3fc7..07a6a2a4 100644 --- a/src/HABApp/openhab/connection/handler/helper.py +++ b/src/HABApp/openhab/connection/handler/helper.py @@ -4,14 +4,17 @@ from typing import Any from HABApp.core.items import BaseValueItem -from HABApp.core.types import RGB, HSB +from HABApp.core.types import HSB, RGB -def convert_to_oh_type(obj: Any) -> str: +def convert_to_oh_type(obj: Any, scientific_floats=False) -> str: if isinstance(obj, (str, int, bool)): return str(obj) if isinstance(obj, float): + if scientific_floats: + return str(obj) + v = str(obj) if 'e-' not in v: return v diff --git a/src/HABApp/openhab/connection/plugins/out.py b/src/HABApp/openhab/connection/plugins/out.py index eaa46fb4..21a2c2e3 100644 --- a/src/HABApp/openhab/connection/plugins/out.py +++ b/src/HABApp/openhab/connection/plugins/out.py @@ -75,6 +75,8 @@ async def queue_worker(self): queue: Final = self.queue to_str: Final = convert_to_oh_type + scientific_floats = not self.plugin_connection.context.workaround_small_floats + while True: try: while True: @@ -84,7 +86,7 @@ async def queue_worker(self): item = item._name if not isinstance(state, str): - state = to_str(state) + state = to_str(state, scientific_floats=scientific_floats) if is_cmd: await post(f'/rest/items/{item:s}', data=state) diff --git a/src/HABApp/openhab/definitions/helpers/persistence_data.py b/src/HABApp/openhab/definitions/helpers/persistence_data.py index 03e78163..5cbc5613 100644 --- a/src/HABApp/openhab/definitions/helpers/persistence_data.py +++ b/src/HABApp/openhab/definitions/helpers/persistence_data.py @@ -3,6 +3,7 @@ from HABApp.openhab.definitions.rest import ItemHistoryResp + OPTIONAL_DT = Optional[datetime] diff --git a/src/HABApp/openhab/items/base_item.py b/src/HABApp/openhab/items/base_item.py index af772e69..bc855c7d 100644 --- a/src/HABApp/openhab/items/base_item.py +++ b/src/HABApp/openhab/items/base_item.py @@ -1,5 +1,5 @@ import datetime -from typing import Any, FrozenSet, Mapping, NamedTuple, Optional, TypeVar, Type +from typing import Any, FrozenSet, Mapping, NamedTuple, Optional, Type from immutables import Map @@ -110,5 +110,4 @@ def get_persistence_data(self, persistence: Optional[str] = None, ) -HINT_OPENHAB_ITEM = TypeVar('HINT_OPENHAB_ITEM', bound=OpenhabItem) -HINT_TYPE_OPENHAB_ITEM = Type[HINT_OPENHAB_ITEM] +HINT_TYPE_OPENHAB_ITEM = Type[OpenhabItem] diff --git a/src/HABApp/openhab/items/commands.py b/src/HABApp/openhab/items/commands.py index 38fa68a7..5452ef4e 100644 --- a/src/HABApp/openhab/items/commands.py +++ b/src/HABApp/openhab/items/commands.py @@ -1,46 +1,47 @@ +from HABApp.core.const.hints import HasNameAttr as _HasNameAttr from HABApp.openhab.definitions import OnOffValue, UpDownValue from HABApp.openhab.interface_sync import send_command class OnOffCommand: - def is_on(self) -> bool: + def is_on(self: _HasNameAttr) -> bool: """Test value against on-value""" raise NotImplementedError() - def is_off(self) -> bool: + def is_off(self: _HasNameAttr) -> bool: """Test value against off-value""" raise NotImplementedError() - def on(self): + def on(self: _HasNameAttr): """Command item on""" - send_command(self, OnOffValue.ON) + send_command(self.name, OnOffValue.ON) - def off(self): + def off(self: _HasNameAttr): """Command item off""" - send_command(self, OnOffValue.OFF) + send_command(self.name, OnOffValue.OFF) class PercentCommand: - def percent(self, value: float): + def percent(self: _HasNameAttr, value: float): """Command to value (in percent)""" assert 0 <= value <= 100, value - send_command(self, str(value)) + send_command(self.name, str(value)) class UpDownCommand: - def up(self): + def up(self: _HasNameAttr): """Command up""" - send_command(self, UpDownValue.UP) + send_command(self.name, UpDownValue.UP) - def down(self): + def down(self: _HasNameAttr): """Command down""" - send_command(self, UpDownValue.DOWN) + send_command(self.name, UpDownValue.DOWN) - def is_up(self) -> bool: + def is_up(self: _HasNameAttr) -> bool: """Test value against on-value""" raise NotImplementedError() - def is_down(self) -> bool: + def is_down(self: _HasNameAttr) -> bool: """Test value against off-value""" raise NotImplementedError() diff --git a/src/HABApp/openhab/items/contact_item.py b/src/HABApp/openhab/items/contact_item.py index be01e1d5..5714b9fc 100644 --- a/src/HABApp/openhab/items/contact_item.py +++ b/src/HABApp/openhab/items/contact_item.py @@ -5,6 +5,7 @@ from ...core.const import MISSING from ..errors import SendCommandNotSupported from HABApp.openhab.interface_sync import post_update +from ...core.errors import InvalidItemValue if TYPE_CHECKING: Optional = Optional @@ -40,8 +41,9 @@ def set_value(self, new_value) -> bool: if isinstance(new_value, OpenClosedValue): new_value = new_value.value - if new_value is not None and new_value != OPEN and new_value != CLOSED: - raise ValueError(f'Invalid value for ContactItem: {new_value}') + if new_value not in (OPEN, CLOSED, None): + raise InvalidItemValue.from_item(self, new_value) + return super().set_value(new_value) def is_open(self) -> bool: diff --git a/src/HABApp/openhab/items/dimmer_item.py b/src/HABApp/openhab/items/dimmer_item.py index 260a3e34..8e419a11 100644 --- a/src/HABApp/openhab/items/dimmer_item.py +++ b/src/HABApp/openhab/items/dimmer_item.py @@ -1,9 +1,12 @@ -from typing import Union, TYPE_CHECKING, Optional, FrozenSet, Mapping +from typing import TYPE_CHECKING, FrozenSet, Mapping, Optional, Union -from HABApp.openhab.items.base_item import OpenhabItem, MetaData +from HABApp.openhab.items.base_item import MetaData, OpenhabItem from HABApp.openhab.items.commands import OnOffCommand, PercentCommand + +from ...core.errors import InvalidItemValue, ItemValueIsNoneError from ..definitions import OnOffValue, PercentValue + if TYPE_CHECKING: Union = Union Optional = Optional @@ -39,15 +42,13 @@ def set_value(self, new_value) -> bool: new_value = new_value.value # Percent is 0 ... 100 - if isinstance(new_value, (int, float)): - assert 0 <= new_value <= 100, new_value - else: - assert new_value is None, new_value + if isinstance(new_value, (int, float)) and (0 <= new_value <= 100): + return super().set_value(new_value) - return super().set_value(new_value) + if new_value is None: + return super().set_value(new_value) - def __str__(self): - return self.value + raise InvalidItemValue.from_item(self, new_value) def is_on(self) -> bool: """Test value against on-value""" @@ -55,4 +56,12 @@ def is_on(self) -> bool: def is_off(self) -> bool: """Test value against off-value""" - return not bool(self.value) + return self.value is not None and not self.value + + def __str__(self): + return self.value + + def __bool__(self): + if self.value is None: + raise ItemValueIsNoneError.from_item(self) + return self.is_on() diff --git a/src/HABApp/openhab/items/number_item.py b/src/HABApp/openhab/items/number_item.py index a3f37b92..4b46969d 100644 --- a/src/HABApp/openhab/items/number_item.py +++ b/src/HABApp/openhab/items/number_item.py @@ -2,6 +2,7 @@ from HABApp.openhab.items.base_item import OpenhabItem, MetaData from ..definitions import QuantityValue +from ...core.errors import ItemValueIsNoneError, InvalidItemValue if TYPE_CHECKING: Union = Union @@ -41,4 +42,15 @@ def set_value(self, new_value) -> bool: if isinstance(new_value, QuantityValue): return super().set_value(new_value.value) - return super().set_value(new_value) + if isinstance(new_value, (int, float)): + return super().set_value(new_value) + + if new_value is None: + return super().set_value(new_value) + + raise InvalidItemValue.from_item(self, new_value) + + def __bool__(self): + if self.value is None: + raise ItemValueIsNoneError.from_item(self) + return bool(self.value) diff --git a/src/HABApp/openhab/items/rollershutter_item.py b/src/HABApp/openhab/items/rollershutter_item.py index 0d892a6f..71717881 100644 --- a/src/HABApp/openhab/items/rollershutter_item.py +++ b/src/HABApp/openhab/items/rollershutter_item.py @@ -1,8 +1,10 @@ -from typing import TYPE_CHECKING, Optional, FrozenSet, Mapping, Union +from typing import TYPE_CHECKING, FrozenSet, Mapping, Optional, Union + +from HABApp.core.errors import InvalidItemValue +from HABApp.openhab.definitions import PercentValue, UpDownValue +from HABApp.openhab.items.base_item import MetaData, OpenhabItem +from HABApp.openhab.items.commands import PercentCommand, UpDownCommand -from HABApp.openhab.items.base_item import OpenhabItem, MetaData -from HABApp.openhab.items.commands import UpDownCommand, PercentCommand -from ..definitions import UpDownValue, PercentValue if TYPE_CHECKING: Union = Union @@ -38,8 +40,14 @@ def set_value(self, new_value) -> bool: elif isinstance(new_value, PercentValue): new_value = new_value.value - assert isinstance(new_value, (int, float)) or new_value is None, new_value - return super().set_value(new_value) + # Position is 0 ... 100 + if isinstance(new_value, (int, float)) and (0 <= new_value <= 100): + return super().set_value(new_value) + + if new_value is None: + return super().set_value(new_value) + + raise InvalidItemValue.from_item(self, new_value) def is_up(self) -> bool: return self.value <= 0 diff --git a/src/HABApp/openhab/items/switch_item.py b/src/HABApp/openhab/items/switch_item.py index a5687f9f..add0db75 100644 --- a/src/HABApp/openhab/items/switch_item.py +++ b/src/HABApp/openhab/items/switch_item.py @@ -1,9 +1,11 @@ -from typing import TYPE_CHECKING, Tuple, Optional, FrozenSet, Mapping +from typing import TYPE_CHECKING, Final, FrozenSet, Mapping, Optional, Tuple +from HABApp.core.errors import ItemValueIsNoneError, InvalidItemValue from HABApp.openhab.definitions import OnOffValue -from HABApp.openhab.items.base_item import OpenhabItem, MetaData +from HABApp.openhab.items.base_item import MetaData, OpenhabItem from HABApp.openhab.items.commands import OnOffCommand + if TYPE_CHECKING: Tuple = Tuple Optional = Optional @@ -12,8 +14,8 @@ MetaData = MetaData -ON = OnOffValue.ON -OFF = OnOffValue.OFF +ON: Final = OnOffValue.ON +OFF: Final = OnOffValue.OFF class SwitchItem(OpenhabItem, OnOffCommand): @@ -28,7 +30,6 @@ class SwitchItem(OpenhabItem, OnOffCommand): :ivar Mapping[str, MetaData] metadata: |oh_item_desc_metadata| """ - @staticmethod def _state_from_oh_str(state: str): if state != ON and state != OFF: @@ -40,8 +41,9 @@ def set_value(self, new_value) -> bool: if isinstance(new_value, OnOffValue): new_value = new_value.value - if new_value is not None and new_value != ON and new_value != OFF: - raise ValueError(f'Invalid value for SwitchItem {self.name}: {new_value}') + if new_value not in (ON, OFF, None): + raise InvalidItemValue.from_item(self, new_value) + return super().set_value(new_value) def is_on(self) -> bool: @@ -52,15 +54,26 @@ def is_off(self) -> bool: """Test value against off-value""" return self.value == OFF + def toggle(self): + """Toggle the switch. Turns the switch on when off or off when currently on.""" + if self.value == ON: + self.off() + elif self.value == OFF: + self.on() + elif self.value is None: + raise ItemValueIsNoneError.from_item(self) + else: + raise InvalidItemValue.from_item(self, self.value) + def __str__(self): return self.value def __eq__(self, other): if isinstance(other, SwitchItem): return self.value == other.value - elif isinstance(other, str): + if isinstance(other, str): return self.value == other - elif isinstance(other, int): + if isinstance(other, int): if other and self.is_on(): return True if not other and self.is_off(): @@ -70,4 +83,6 @@ def __eq__(self, other): return NotImplemented def __bool__(self): - return self.is_on() + if self.value is None: + raise ItemValueIsNoneError.from_item(self) + return self.value == ON diff --git a/src/HABApp/openhab/map_items.py b/src/HABApp/openhab/map_items.py index d08ed681..a65db9ef 100644 --- a/src/HABApp/openhab/map_items.py +++ b/src/HABApp/openhab/map_items.py @@ -3,13 +3,14 @@ from immutables import Map -import HABApp from HABApp.core.wrapper import process_exception from HABApp.openhab.definitions.values import QuantityValue -from HABApp.openhab.items import ColorItem, ContactItem, DatetimeItem, DimmerItem, GroupItem, ImageItem, LocationItem, \ - NumberItem, PlayerItem, RollershutterItem, StringItem, SwitchItem, CallItem -from HABApp.openhab.items.base_item import HINT_TYPE_OPENHAB_ITEM -from HABApp.openhab.items.base_item import MetaData +from HABApp.openhab.items import ( + CallItem, ColorItem, ContactItem, DatetimeItem, DimmerItem, GroupItem, ImageItem, + LocationItem, NumberItem, PlayerItem, RollershutterItem, StringItem, SwitchItem, +) +from HABApp.openhab.items.base_item import HINT_TYPE_OPENHAB_ITEM, MetaData, OpenhabItem + log = logging.getLogger('HABApp.openhab.items') @@ -34,7 +35,7 @@ def map_item(name: str, type: str, value: Optional[str], label: Optional[str], tags: FrozenSet[str], groups: FrozenSet[str], metadata: Optional[Dict[str, Dict[str, Any]]]) -> \ - Optional['HABApp.openhab.items.OpenhabItem']: + Optional[OpenhabItem]: try: assert isinstance(type, str) assert value is None or isinstance(value, str) diff --git a/src/HABApp/util/__init__.py b/src/HABApp/util/__init__.py index 2b09100d..8b56fd5a 100644 --- a/src/HABApp/util/__init__.py +++ b/src/HABApp/util/__init__.py @@ -2,6 +2,7 @@ from .statistics import Statistics from .listener_groups import EventListenerGroup from .fade import Fade +from .rate_limiter import RateLimiter from . import functions from . import multimode diff --git a/src/HABApp/util/functions/min_max.py b/src/HABApp/util/functions/min_max.py index d03284dc..92ab567b 100644 --- a/src/HABApp/util/functions/min_max.py +++ b/src/HABApp/util/functions/min_max.py @@ -2,8 +2,9 @@ from builtins import min as _min -def max(*args, default=None): - """Behaves like the built in max function but ignores any ``None`` values. e.g. ``max([1, None, 2]) == 2``. +# noinspection PyShadowingBuiltins +def max(*args, default=None): # noqa: A001 + """Behaves like the built-in max function but ignores any ``None`` values. e.g. ``max([1, None, 2]) == 2``. If the iterable is empty ``default`` will be returned. :param args: Single iterable or 1..n arguments @@ -16,8 +17,9 @@ def max(*args, default=None): ) -def min(*args, default=None): - """Behaves like the built in min function but ignores any ``None`` values. e.g. ``min([1, None, 2]) == 1``. +# noinspection PyShadowingBuiltins +def min(*args, default=None): # noqa: A001 + """Behaves like the built-in min function but ignores any ``None`` values. e.g. ``min([1, None, 2]) == 1``. If the iterable is empty ``default`` will be returned. :param args: Single iterable or 1..n arguments diff --git a/src/HABApp/util/rate_limiter/__init__.py b/src/HABApp/util/rate_limiter/__init__.py new file mode 100644 index 00000000..558dc8ea --- /dev/null +++ b/src/HABApp/util/rate_limiter/__init__.py @@ -0,0 +1 @@ +from .registry import RateLimiter diff --git a/src/HABApp/util/rate_limiter/limiter.py b/src/HABApp/util/rate_limiter/limiter.py new file mode 100644 index 00000000..7b6aaead --- /dev/null +++ b/src/HABApp/util/rate_limiter/limiter.py @@ -0,0 +1,158 @@ +from dataclasses import dataclass +from typing import Final, List, Literal, Tuple, Union, get_args + +from HABApp.core.const.const import PYTHON_310 +from HABApp.util.rate_limiter.limits import BaseRateLimit, FixedWindowElasticExpiryLimit, \ + FixedWindowElasticExpiryLimitInfo, LeakyBucketLimit, LeakyBucketLimitInfo +from HABApp.util.rate_limiter.parser import parse_limit + +if PYTHON_310: + from typing import TypeAlias +else: + from typing_extensions import TypeAlias + + +_LITERAL_LEAKY_BUCKET = Literal['leaky_bucket'] +_LITERAL_FIXED_WINDOW_ELASTIC_EXPIRY = Literal['fixed_window_elastic_expiry'] + +LIMITER_ALGORITHM_HINT: TypeAlias = Literal[_LITERAL_LEAKY_BUCKET, _LITERAL_FIXED_WINDOW_ELASTIC_EXPIRY] + + +def _check_arg(name: str, value, allow_0=False): + if not isinstance(value, int) or ((value <= 0) if not allow_0 else (value < 0)): + msg = f'Parameter {name:s} must be an int >{"=" if allow_0 else ""} 0, is {value} ({type(value)})' + raise ValueError(msg) + + +class Limiter: + def __init__(self, name: str): + self._name: Final = name + self._limits: Tuple[BaseRateLimit, ...] = () + self._skips: int = 0 + self._skips_total: int = 0 + + @property + def total_skips(self) -> int: + """A counter to track skips which can be manually reset""" + return self._skips_total + + def __repr__(self): + return f'<{self.__class__.__name__} {self._name:s}>' + + def add_limit(self, allowed: int, interval: int, *, + initial_hits: int = 0, + algorithm: LIMITER_ALGORITHM_HINT = 'leaky_bucket') -> 'Limiter': + """Add a new rate limit + + :param allowed: How many hits are allowed + :param interval: Interval in seconds + :param initial_hits: How many hits the limit already has when it gets initially created + :param algorithm: Which algorithm should this limit use + """ + _check_arg('allowed', allowed) + _check_arg('interval', interval) + _check_arg('hits', initial_hits, allow_0=True) + if not initial_hits <= allowed: + msg = f'Parameter hits must be <= parameter allowed! {initial_hits:d} <= {allowed:d}!' + raise ValueError(msg) + + if algorithm == get_args(_LITERAL_LEAKY_BUCKET)[0]: + cls = LeakyBucketLimit + elif algorithm == get_args(_LITERAL_FIXED_WINDOW_ELASTIC_EXPIRY)[0]: + cls = FixedWindowElasticExpiryLimit + else: + msg = f'Unknown algorithm "{algorithm}"' + raise ValueError(msg) + + # Check if we have already added an algorithm with these parameters + for window in self._limits: + if isinstance(window, cls) and window.allowed == allowed and window.interval == interval: + return self + + limit = cls(allowed, interval, hits=initial_hits) + self._limits = tuple(sorted([*self._limits, limit], key=lambda x: x.interval)) + return self + + def parse_limits(self, *text: str, + initial_hits: int = 0, + algorithm: LIMITER_ALGORITHM_HINT = 'leaky_bucket') -> 'Limiter': + """Add one or more limits in textual form, e.g. ``5 in 60s``, ``10 per hour`` or ``10/15 mins``. + If the limit does already exist it will not be added again. + + :param text: textual description of limit + :param initial_hits: How many hits the limit already has when it gets initially created + :param algorithm: Which algorithm should these limits use + """ + for limit in [parse_limit(t) for t in text]: + self.add_limit(*limit, initial_hits=initial_hits, algorithm=algorithm) + return self + + def allow(self) -> bool: + """Test the limit(s). + + :return: ``True`` if allowed, ``False`` if forbidden + """ + if not self._limits: + msg = 'No limits defined!' + raise ValueError(msg) + + clear_skipped = True + + for limit in self._limits: + if not limit.allow(): + self._skips += 1 + self._skips_total += 1 + return False + + # allow increments hits, if it's now 1 it was 0 before + if limit.hits != 1: + clear_skipped = False + + if clear_skipped: + self._skips = 0 + + return True + + def test_allow(self) -> bool: + """Test the limit(s) without hitting it. Calling this will not increase the hit counter. + + :return: ``True`` if allowed, ``False`` if forbidden + """ + + if not self._limits: + msg = 'No limits defined!' + raise ValueError(msg) + + clear_skipped = True + + for limit in self._limits: + if not limit.test_allow(): + return False + + if limit.hits != 0: + clear_skipped = False + + if clear_skipped: + self._skips = 0 + return True + + def info(self) -> 'LimiterInfo': + """Get some info about the limiter and the defined windows + """ + + return LimiterInfo( + skips=self._skips, total_skips=self._skips_total, + limits=[limit.info() for limit in self._limits] + ) + + def reset(self) -> 'Limiter': + """Reset the skip counter""" + self._skips_total = 0 + return self + + +@dataclass +class LimiterInfo: + skips: int #: How many entries were skipped in the active interval(s) + total_skips: int #: How many entries were skipped in total + limits: List[Union[FixedWindowElasticExpiryLimitInfo, LeakyBucketLimitInfo]] #: Info for every limit diff --git a/src/HABApp/util/rate_limiter/limits/__init__.py b/src/HABApp/util/rate_limiter/limits/__init__.py new file mode 100644 index 00000000..2d7e52ce --- /dev/null +++ b/src/HABApp/util/rate_limiter/limits/__init__.py @@ -0,0 +1,3 @@ +from .base import BaseRateLimit +from .fixed_window import FixedWindowElasticExpiryLimit, FixedWindowElasticExpiryLimitInfo +from .leaky_bucket import LeakyBucketLimit, LeakyBucketLimitInfo diff --git a/src/HABApp/util/rate_limiter/limits/base.py b/src/HABApp/util/rate_limiter/limits/base.py new file mode 100644 index 00000000..ef8ae87d --- /dev/null +++ b/src/HABApp/util/rate_limiter/limits/base.py @@ -0,0 +1,74 @@ +from dataclasses import dataclass +from typing import Final + + +@dataclass +class BaseRateLimitInfo: + hits: int #: Hits + skips: int #: Skips + limit: int #: Boundary + + @property + def hits_remaining(self) -> int: + return self.limit - self.hits + + +class BaseRateLimit: + def __init__(self, allowed: int, interval: int, hits: int = 0): + super().__init__() + assert allowed > 0, allowed + assert interval > 0, interval + assert 0 <= hits <= allowed + + self.interval: Final = interval + self.allowed: Final = allowed + + self.hits: int = hits + self.skips: int = 0 + + def repr_text(self) -> str: + return '' + + def __repr__(self): + return ( + f'<{self.__class__.__name__} hits={self.hits:d}/{self.allowed:d} interval={self.interval:d}s ' + f'{self.repr_text():s}>' + ) + + def do_test_allow(self): + raise NotImplementedError() + + def do_allow(self): + raise NotImplementedError() + + def do_deny(self): + raise NotImplementedError() + + def info(self) -> BaseRateLimitInfo: + raise NotImplementedError() + + def allow(self, weight: int = 1) -> bool: + if not isinstance(weight, int) or weight <= 0: + msg = f'weight must be an int > 0, is {weight}' + raise ValueError(msg) + + self.do_allow() + + self.hits += weight + if self.hits <= self.allowed: + return True + + self.skips += 1 + self.hits = self.allowed + + if self.do_deny: + self.do_deny() + return False + + def test_allow(self, weight: int = 1) -> bool: + if not isinstance(weight, int) or weight <= 0: + msg = f'weight must be an int > 0, is {weight}' + raise ValueError(msg) + + self.do_test_allow() + return self.hits + weight <= self.allowed diff --git a/src/HABApp/util/rate_limiter/limits/fixed_window.py b/src/HABApp/util/rate_limiter/limits/fixed_window.py new file mode 100644 index 00000000..95cc4328 --- /dev/null +++ b/src/HABApp/util/rate_limiter/limits/fixed_window.py @@ -0,0 +1,51 @@ +from dataclasses import dataclass +from time import monotonic + +from .base import BaseRateLimit, BaseRateLimitInfo + + +@dataclass +class FixedWindowElasticExpiryLimitInfo(BaseRateLimitInfo): + time_remaining: float #: Time remaining until this window will reset + + +class FixedWindowElasticExpiryLimit(BaseRateLimit): + def __init__(self, allowed: int, interval: int, hits: int = 0): + super().__init__(allowed, interval, hits) + + self.start: float = -1.0 + self.stop: float = -1.0 + + def repr_text(self): + return f'window={self.stop - self.start:.0f}s' + + def do_test_allow(self): + if self.stop <= monotonic(): + self.hits = 0 + self.skips = 0 + + def do_allow(self): + now = monotonic() + + if self.stop <= now: + self.hits = 0 + self.skips = 0 + self.start = now + self.stop = now + self.interval + + def do_deny(self): + self.stop = monotonic() + self.interval + + def info(self) -> FixedWindowElasticExpiryLimitInfo: + self.do_test_allow() + + remaining = self.stop - monotonic() + if remaining <= 0: + remaining = 0 + + if not remaining and not self.hits: + remaining = self.interval + + return FixedWindowElasticExpiryLimitInfo( + time_remaining=remaining, hits=self.hits, skips=self.skips, limit=self.allowed + ) diff --git a/src/HABApp/util/rate_limiter/limits/leaky_bucket.py b/src/HABApp/util/rate_limiter/limits/leaky_bucket.py new file mode 100644 index 00000000..fe39bb2e --- /dev/null +++ b/src/HABApp/util/rate_limiter/limits/leaky_bucket.py @@ -0,0 +1,50 @@ +from dataclasses import dataclass +from time import monotonic +from typing import Final + +from .base import BaseRateLimit, BaseRateLimitInfo + + +@dataclass +class LeakyBucketLimitInfo(BaseRateLimitInfo): + time_remaining: float #: Time remaining until the next drop + + +class LeakyBucketLimit(BaseRateLimit): + def __init__(self, allowed: int, interval: int, hits: int = 0): + super().__init__(allowed, interval, hits) + + self.drop_interval: Final = interval / allowed + self.next_drop: float = monotonic() + self.drop_interval + + def repr_text(self): + return f'drop_interval={self.drop_interval:.1f}s' + + def do_test_allow(self): + + while self.next_drop <= monotonic(): + self.hits -= 1 + self.next_drop += self.drop_interval + + if self.hits <= 0: + # out of drop interval -> reset stats + if self.next_drop <= monotonic(): + self.next_drop = monotonic() + self.drop_interval + self.skips = 0 + + self.hits = 0 + break + + do_allow = do_test_allow + do_deny = None + + def info(self) -> LeakyBucketLimitInfo: + self.do_test_allow() + + remaining = self.next_drop - monotonic() + if remaining <= 0: + remaining = 0 + + return LeakyBucketLimitInfo( + time_remaining=remaining, hits=self.hits, skips=self.skips, limit=self.allowed + ) diff --git a/src/HABApp/util/rate_limiter/parser.py b/src/HABApp/util/rate_limiter/parser.py new file mode 100644 index 00000000..44e8b4a1 --- /dev/null +++ b/src/HABApp/util/rate_limiter/parser.py @@ -0,0 +1,27 @@ +import re +from typing import Tuple + +LIMIT_REGEX = re.compile( + r""" + \s* ([1-9][0-9]*) + \s* (/|per|in) + \s* ([1-9][0-9]*)? + \s* (s|sec|second|m|min|minute|h|hour|day|month|year)s? + \s*""", + re.IGNORECASE | re.VERBOSE, +) + + +def parse_limit(text: str) -> Tuple[int, int]: + if not isinstance(text, str) or not (m := LIMIT_REGEX.fullmatch(text)): + msg = f'Invalid limit string: "{text:s}"' + raise ValueError(msg) + + count, per, factor, interval = m.groups() + + interval_secs = { + 's': 1, 'sec': 1, 'second': 1, 'm': 60, 'min': 60, 'minute': 60, 'hour': 3600, 'h': 3600, + 'day': 24 * 3600, 'month': 30 * 24 * 3600, 'year': 365 * 24 * 3600 + }[interval] + + return int(count), int(1 if factor is None else factor) * interval_secs diff --git a/src/HABApp/util/rate_limiter/registry.py b/src/HABApp/util/rate_limiter/registry.py new file mode 100644 index 00000000..048c7e40 --- /dev/null +++ b/src/HABApp/util/rate_limiter/registry.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from threading import Lock + +from .limiter import Limiter + + +LOCK = Lock() + +_LIMITERS: dict[str, Limiter] = {} + + +def RateLimiter(name: str) -> Limiter: + """Create a new rate limiter or return an already existing one with a given name. + + :param name: case insensitive name of limiter + :return: Rate limiter object + """ + + key = name.lower() + + with LOCK: + if (obj := _LIMITERS.get(key)) is None: + _LIMITERS[key] = obj = Limiter(name) + + return obj diff --git a/tests/test_core/test_types/test_color.py b/tests/test_core/test_types/test_color.py index f464a86a..375b04f5 100644 --- a/tests/test_core/test_types/test_color.py +++ b/tests/test_core/test_types/test_color.py @@ -1,6 +1,6 @@ import pytest -from HABApp.core.types.color import RGB, HSB +from HABApp.core.types.color import HSB, RGB def test_rgb(): @@ -10,8 +10,16 @@ def test_rgb(): assert rgb.b == rgb.blue == 3 assert rgb[0] == 1 + assert rgb['r'] == 1 + assert rgb['red'] == 1 + assert rgb[1] == 2 + assert rgb['g'] == 2 + assert rgb['green'] == 2 + assert rgb[2] == 3 + assert rgb['b'] == 3 + assert rgb['blue'] == 3 r, g, b = rgb assert r == rgb.r @@ -69,8 +77,16 @@ def test_hsb(): assert hsb.b == hsb.brightness == 3 assert hsb[0] == 1 + assert hsb['h'] == 1 + assert hsb['hue'] == 1 + assert hsb[1] == 2 + assert hsb['s'] == 2 + assert hsb['saturation'] == 2 + assert hsb[2] == 3 + assert hsb['b'] == 3 + assert hsb['brightness'] == 3 h, s, b = hsb assert h == hsb.h diff --git a/tests/test_openhab/test_items/test_commands.py b/tests/test_openhab/test_items/test_commands.py index 599517ca..99de6c8c 100644 --- a/tests/test_openhab/test_items/test_commands.py +++ b/tests/test_openhab/test_items/test_commands.py @@ -2,15 +2,20 @@ import pytest -from HABApp.openhab.definitions import OnOffValue, UpDownValue, OpenClosedValue +from HABApp import __version__ +from HABApp.openhab.definitions import OnOffValue, OpenClosedValue, UpDownValue from HABApp.openhab.items import ContactItem -from HABApp.openhab.items.commands import UpDownCommand, OnOffCommand +from HABApp.openhab.items.commands import OnOffCommand, UpDownCommand from HABApp.openhab.map_items import _items as item_dict @pytest.mark.parametrize("cls", [cls for cls in item_dict.values() if issubclass(cls, OnOffCommand)]) def test_OnOff(cls): c = cls('item_name') + assert not c.is_on() + if not __version__.startswith('24.01.0'): + assert not c.is_off() + c.set_value(OnOffValue('ON')) assert c.is_on() assert not c.is_off() @@ -37,6 +42,9 @@ def test_UpDown(cls): @pytest.mark.parametrize("cls", (ContactItem, )) def test_OpenClosed(cls: typing.Type[ContactItem]): c = cls('item_name') + assert not c.is_closed() + assert not c.is_open() + c.set_value(OpenClosedValue.OPEN) assert c.is_open() assert not c.is_closed() diff --git a/tests/test_openhab/test_items/test_contact.py b/tests/test_openhab/test_items/test_contact.py index f551a4f8..638e4661 100644 --- a/tests/test_openhab/test_items/test_contact.py +++ b/tests/test_openhab/test_items/test_contact.py @@ -1,5 +1,6 @@ import pytest +from HABApp.core.errors import InvalidItemValue from HABApp.openhab.errors import SendCommandNotSupported from HABApp.openhab.items import ContactItem @@ -11,3 +12,12 @@ def test_send_command(): c.oh_send_command('asdf') assert str(e.value) == 'ContactItem does not support send command! See openHAB documentation for details.' + + +def test_switch_set_value(): + ContactItem('').set_value(None) + ContactItem('').set_value('OPEN') + ContactItem('').set_value('CLOSED') + + with pytest.raises(InvalidItemValue): + ContactItem('item_name').set_value('asdf') diff --git a/tests/test_openhab/test_items/test_dimmer.py b/tests/test_openhab/test_items/test_dimmer.py new file mode 100644 index 00000000..fee6a32b --- /dev/null +++ b/tests/test_openhab/test_items/test_dimmer.py @@ -0,0 +1,22 @@ +import pytest + +from HABApp.core.errors import InvalidItemValue, ItemValueIsNoneError +from HABApp.openhab.items import DimmerItem + + +def test_dimmer_item_bool(): + with pytest.raises(ItemValueIsNoneError): + assert not DimmerItem('asdf') + + assert not DimmerItem('asdf', 0) + assert DimmerItem('asdf', 1) + + +def test_dimmer_set_value(): + DimmerItem('').set_value(None) + DimmerItem('').set_value(0) + DimmerItem('').set_value(100) + DimmerItem('').set_value(55.55) + + with pytest.raises(InvalidItemValue): + DimmerItem('item_name').set_value('asdf') diff --git a/tests/test_openhab/test_items/test_number.py b/tests/test_openhab/test_items/test_number.py index efecef75..a9b75a51 100644 --- a/tests/test_openhab/test_items/test_number.py +++ b/tests/test_openhab/test_items/test_number.py @@ -1,5 +1,7 @@ +import pytest from immutables import Map +from HABApp.core.errors import InvalidItemValue, ItemValueIsNoneError from HABApp.openhab.items import NumberItem from HABApp.openhab.items.base_item import MetaData @@ -7,3 +9,20 @@ def test_number_item_unit(): assert NumberItem('test', 1).unit is None assert NumberItem('test', 1, metadata=Map(unit=MetaData('°C'))).unit == '°C' + + +def test_number_item_bool(): + with pytest.raises(ItemValueIsNoneError): + assert not NumberItem('asdf') + + assert not NumberItem('asdf', 0) + assert NumberItem('asdf', 1) + + +def test_number_set_value(): + NumberItem('').set_value(None) + NumberItem('').set_value(1) + NumberItem('').set_value(-3.3) + + with pytest.raises(InvalidItemValue): + NumberItem('item_name').set_value('asdf') diff --git a/tests/test_openhab/test_items/test_rollershutter.py b/tests/test_openhab/test_items/test_rollershutter.py new file mode 100644 index 00000000..a6b0938c --- /dev/null +++ b/tests/test_openhab/test_items/test_rollershutter.py @@ -0,0 +1,14 @@ +import pytest + +from HABApp.core.errors import InvalidItemValue +from HABApp.openhab.items import RollershutterItem + + +def test_dimmer_set_value(): + RollershutterItem('').set_value(None) + RollershutterItem('').set_value(0) + RollershutterItem('').set_value(100) + RollershutterItem('').set_value(55.55) + + with pytest.raises(InvalidItemValue): + RollershutterItem('item_name').set_value('asdf') diff --git a/tests/test_openhab/test_items/test_switch.py b/tests/test_openhab/test_items/test_switch.py new file mode 100644 index 00000000..6bc7c9f1 --- /dev/null +++ b/tests/test_openhab/test_items/test_switch.py @@ -0,0 +1,21 @@ +import pytest + +from HABApp.core.errors import InvalidItemValue, ItemValueIsNoneError +from HABApp.openhab.items import SwitchItem + + +def test_switch_item_bool(): + with pytest.raises(ItemValueIsNoneError): + assert SwitchItem('test') + + assert not SwitchItem('test', 'OFF') + assert SwitchItem('test', 'ON') + + +def test_switch_set_value(): + SwitchItem('').set_value(None) + SwitchItem('').set_value('ON') + SwitchItem('').set_value('OFF') + + with pytest.raises(InvalidItemValue): + SwitchItem('item_name').set_value('asdf') diff --git a/tests/test_openhab/test_plugins/test_load_items.py b/tests/test_openhab/test_plugins/test_load_items.py index 8a124adf..d548da7d 100644 --- a/tests/test_openhab/test_plugins/test_load_items.py +++ b/tests/test_openhab/test_plugins/test_load_items.py @@ -78,7 +78,9 @@ async def test_item_sync(monkeypatch, ir: ItemRegistry, test_logs): waited_for_openhab=False, created_items={}, created_things={}, - session=None, session_options=None + session=None, session_options=None, + + workaround_small_floats=False ) # initial item create await LoadOpenhabItemsPlugin().on_connected(context) diff --git a/tests/test_utils/test_rate_limiter.py b/tests/test_utils/test_rate_limiter.py new file mode 100644 index 00000000..844bb72d --- /dev/null +++ b/tests/test_utils/test_rate_limiter.py @@ -0,0 +1,251 @@ +import re + +import pytest + +import HABApp.util.rate_limiter.limits.fixed_window as fixed_window_module +import HABApp.util.rate_limiter.limits.leaky_bucket as leaky_bucket_module +import HABApp.util.rate_limiter.registry as registry_module +from HABApp.util.rate_limiter.limiter import ( + FixedWindowElasticExpiryLimit, + FixedWindowElasticExpiryLimitInfo, + LeakyBucketLimit, + LeakyBucketLimitInfo, + Limiter, + parse_limit, +) +from HABApp.util.rate_limiter.parser import LIMIT_REGEX + + +class MockedMonotonic: + def __init__(self): + self.time = 0 + + def get_time(self): + return self.time + + def __iadd__(self, other): + self.time += other + return self + + +@pytest.fixture() +def time(monkeypatch) -> MockedMonotonic: + m = MockedMonotonic() + monkeypatch.setattr(fixed_window_module, 'monotonic', m.get_time) + monkeypatch.setattr(leaky_bucket_module, 'monotonic', m.get_time) + return m + + +@pytest.mark.parametrize( + 'unit,factor', [ + ('s', 1), ('sec', 1), ('second', 1), + ('m', 60), ('min', 60), ('minute', 60), + ('h', 3600), ('hour', 3600), + ('day', 24 * 3600), ('month', 30 * 24 * 3600), ('year', 365 * 24 * 3600) + ] +) +def test_parse(unit: str, factor: int): + assert parse_limit(f' 1 per {unit} ') == (1, factor) + assert parse_limit(f' 1 / {unit} ') == (1, factor) + assert parse_limit(f'3 per {unit}') == (3, factor) + assert parse_limit(f'3 in {unit}') == (3, factor) + + for ctr in (1, 12, 375, 5533): + assert parse_limit(f'{ctr:d} in 5{unit}') == (ctr, 5 * factor) + assert parse_limit(f'{ctr:d} in 5{unit}s') == (ctr, 5 * factor) + + with pytest.raises(ValueError) as e: + parse_limit('asdf') + assert str(e.value) == 'Invalid limit string: "asdf"' + + +def test_parse_regex_all_units(): + m = re.search(r'\(([^)]+)\)s\?', LIMIT_REGEX.pattern) + values = m.group(1) + + for unit in values.split('|'): + parse_limit(f'1 in 3 {unit}') + parse_limit(f'1 in 3 {unit}s') + + +def test_fixed_window(time): + + limit = FixedWindowElasticExpiryLimit(5, 3) + assert str(limit) == '' + for _ in range(10): + assert limit.test_allow() + + assert limit.allow() + assert str(limit) == '' + + for _ in range(4): + assert limit.allow() + + assert str(limit) == '' + + # Limit is full, stop gets moved further + time += 1 + assert not limit.allow() + assert str(limit) == '' + + # move out of interval + time += 3.1 + assert limit.allow() + assert limit.hits == 1 + assert str(limit) == '' + + +def test_leaky_bucket(time): + limit = LeakyBucketLimit(4, 2) + assert str(limit) == '' + for _ in range(10): + assert limit.test_allow() + + assert limit.allow() + assert limit.hits == 1 + + assert limit.allow() + assert limit.hits == 2 + + assert limit.allow() + assert limit.allow() + assert not limit.allow() + assert not limit.allow() + assert limit.hits == 4 + + time += 0.5 + assert limit.test_allow() + assert limit.hits == 3 + + time += 1.7 + assert limit.test_allow() + assert limit.hits == 0 + + assert limit.allow() + assert limit.hits == 1 + + time += 0.3 + assert limit.test_allow() + assert limit.hits == 0 + + time += 1 + assert limit.allow() + time += 0.4999 + + limit.test_allow() + assert limit.hits == 1 + + time += 0.0001 + limit.test_allow() + assert limit.hits == 0 + + +def test_window_test_allow(time): + + limit = FixedWindowElasticExpiryLimit(5, 3) + limit.hits = 5 + limit.stop = 2.99999 + assert not limit.test_allow() + + # expiry when out of window + time += 3 + assert limit.test_allow() + assert not limit.hits + + +def test_limiter_add(time): + limiter = Limiter('test') + limiter.add_limit(3, 5).add_limit(3, 5).parse_limits('3 in 5s') + assert len(limiter._limits) == 1 + + with pytest.raises(ValueError) as e: + limiter.add_limit(0, 5) + assert str(e.value) == "Parameter allowed must be an int > 0, is 0 ()" + + with pytest.raises(ValueError) as e: + limiter.add_limit(1, 0.5) + assert str(e.value) == "Parameter interval must be an int > 0, is 0.5 ()" + + with pytest.raises(ValueError) as e: + limiter.add_limit(3, 5, initial_hits=-1) + assert str(e.value) == "Parameter hits must be an int >= 0, is -1 ()" + + with pytest.raises(ValueError) as e: + limiter.add_limit(3, 5, initial_hits=5) + assert str(e.value) == "Parameter hits must be <= parameter allowed! 5 <= 3!" + + +def test_fixed_window_info(time): + limit = FixedWindowElasticExpiryLimit(5, 3) + Info = FixedWindowElasticExpiryLimitInfo + + assert limit.info() == Info(hits=0, skips=0, limit=5, time_remaining=3) + + limit.allow() + assert limit.info() == Info(hits=1, skips=0, limit=5, time_remaining=3) + limit.allow(4) + assert limit.info() == Info(hits=5, skips=0, limit=5, time_remaining=3) + limit.allow() + assert limit.info() == Info(hits=5, skips=1, limit=5, time_remaining=3) + + time += 1 + assert limit.info() == Info(hits=5, skips=1, limit=5, time_remaining=2) + + time += 3 + assert limit.info() == Info(hits=0, skips=0, limit=5, time_remaining=3) + + assert not limit.test_allow(6) + assert limit.info() == Info(hits=0, skips=0, limit=5, time_remaining=3) + + +def test_leaky_bucket_info(time): + limit = LeakyBucketLimit(2, 2) + Info = LeakyBucketLimitInfo + + assert limit.info() == Info(hits=0, skips=0, limit=2, time_remaining=1) + + +def test_registry(monkeypatch): + monkeypatch.setattr(registry_module, '_LIMITERS', {}) + + obj = registry_module.RateLimiter('Test') + assert obj is registry_module.RateLimiter('TEST') + assert obj is registry_module.RateLimiter('test') + + +def test_limiter(time): + + limiter = Limiter('Test') + assert limiter.__repr__() == '' + + info = limiter.info() + assert info.skips == 0 + + with pytest.raises(ValueError): + limiter.allow() + + limiter.add_limit( + 2, 1, algorithm='fixed_window_elastic_expiry').add_limit(2, 2, algorithm='fixed_window_elastic_expiry') + + assert limiter.allow() + assert limiter.allow() + time += 0.5 + assert not limiter.allow() + + time += 1 + assert not limiter.allow() + + assert limiter.info().skips == 2 + time += 2 + + assert limiter.test_allow() + assert limiter.info().skips == 0 + + assert limiter.allow() + assert limiter.allow() + assert not limiter.allow() + assert limiter.info().skips == 1 + + time += 2 + assert limiter.allow() + assert limiter.info().skips == 0