From feb5a2fc1d2e47a53db8d8913e8f69bfac6804ea Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Fri, 1 Dec 2023 11:32:47 +0100 Subject: [PATCH] 23.12.0-DEV --- docs/logging.rst | 17 +- docs/requirements.txt | 6 +- docs/util.rst | 82 +++++++++ readme.md | 7 +- requirements_setup.txt | 2 +- run/conf/logging.yml | 8 +- src/HABApp/__version__.py | 2 +- src/HABApp/config/logging/__init__.py | 2 +- src/HABApp/config/logging/default_logfile.py | 2 + src/HABApp/config/logging/handler.py | 26 +++ src/HABApp/util/__init__.py | 1 + src/HABApp/util/rate_limiter/__init__.py | 1 + src/HABApp/util/rate_limiter/limiter.py | 143 ++++++++++++++++ src/HABApp/util/rate_limiter/rate_limit.py | 73 ++++++++ src/HABApp/util/rate_limiter/registry.py | 26 +++ tests/test_utils/test_rate_limiter.py | 166 +++++++++++++++++++ 16 files changed, 551 insertions(+), 13 deletions(-) create mode 100644 src/HABApp/util/rate_limiter/__init__.py create mode 100644 src/HABApp/util/rate_limiter/limiter.py create mode 100644 src/HABApp/util/rate_limiter/rate_limit.py create mode 100644 src/HABApp/util/rate_limiter/registry.py create mode 100644 tests/test_utils/test_rate_limiter.py diff --git a/docs/logging.rst b/docs/logging.rst index 297fe85d..55f4af58 100644 --- a/docs/logging.rst +++ b/docs/logging.rst @@ -17,6 +17,17 @@ but the format should be pretty straight forward. | That way even if the HABApp configuration is invalid HABApp can still log the errors that have occurred. | e.g.: ``/HABApp/logs/habapp.log`` or ``c:\HABApp\logs\habapp.log`` +Provided loggers +====================================== + +The ``HABApp.config.logging`` module provides additional loggers which can be used + + +.. autoclass:: HABApp.config.logging.MidnightRotatingFileHandler + +.. autoclass:: HABApp.config.logging.CompressedMidnightRotatingFileHandler + + Example ====================================== @@ -42,7 +53,7 @@ to the file configuration under ``handlers`` in the ``logging.yml``. ... MyRuleHandler: # <-- This is the name of the handler - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.MidnightRotatingFileHandler filename: 'c:\HABApp\Logs\MyRule.log' maxBytes: 10_000_000 backupCount: 3 @@ -84,7 +95,7 @@ Full Example configuration # ----------------------------------------------------------------------------------- handlers: HABApp_default: - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.MidnightRotatingFileHandler filename: 'HABApp.log' maxBytes: 10_000_000 backupCount: 3 @@ -93,7 +104,7 @@ Full Example configuration level: DEBUG MyRuleHandler: - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.MidnightRotatingFileHandler filename: 'c:\HABApp\Logs\MyRule.log' # absolute filename is recommended maxBytes: 10_000_000 backupCount: 3 diff --git a/docs/requirements.txt b/docs/requirements.txt index b3f65219..8032d1f4 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ # Packages required to build the documentation -sphinx == 7.2.5 -sphinx-autodoc-typehints == 1.24.0 -sphinx_rtd_theme == 1.3.0 +sphinx == 7.2.6 +sphinx-autodoc-typehints == 1.25.2 +sphinx_rtd_theme == 2.0.0 sphinx-exec-code == 0.10 autodoc_pydantic == 2.0.1 sphinx-copybutton == 0.5.2 diff --git a/docs/util.rst b/docs/util.rst index 4e5f04ed..53f4c663 100644 --- a/docs/util.rst +++ b/docs/util.rst @@ -75,6 +75,88 @@ Converts a hsb value to the rgb color space .. autofunction:: HABApp.util.functions.hsb_to_rgb +Rate limiter +------------------------------ +A simple rate limiter implementation which can be used in rules. +The limiter is not rule bound so the same limiter can be used in multiples files. +It also works as expected across rule reloads. + +Defining limits +^^^^^^^^^^^^^^^^^^ +Limits can either be explicitly added or through a textual description. +If the limit does already exist it will not be added again. +It's possible to explicitly create the limits or through some small textual description with the following syntax: + +.. code-block:: text + + [count] [per|in|/] [count (optional)] [s|sec|second|m|min|minute|hour|h|day|month|year] [s (optional)] + +Whitespaces are ignored and can be added as desired + +Examples: + +* ``5 per minute`` +* ``20 in 15 mins`` +* ``300 / hour`` + + +Elastic expiry +^^^^^^^^^^^^^^^^^^ + +The rate limiter implements a fixed window with elastic expiry. +That means if the limit is hit the interval time will be increased by the expiry time. + +For example ``3 per minute``: + +* First hit comes ``00:00:00``. Two more hits at ``00:00:59``. + All three pass, intervall goes from ``00:00:00`` - ``00:01:00``. + Another hit comes at ``00:01:01`` an passes. The intervall now goes from ``00:01:01`` - ``00:02:01``. + +* First hit comes ``00:00:00``. Two more hits at ``00:00:30``. All three pass. + Another hit comes at ``00:00:45``, which gets rejected and the intervall now goes from ``00:00:00`` - ``00:01:45``. + A rejected hit makes the interval time longer by expiry time. If another hit comes at ``00:01:30`` it + will also get rejected and the intervall now goes from ``00:00:00`` - ``00:02:30``. + + +Example +^^^^^^^^^^^^^^^^^^ + +.. exec_code:: + + from HABApp.util import RateLimiter + + # Create or get existing, name is case insensitive + limiter = RateLimiter('MyRateLimiterName') + + # define limits, duplicate limits will only be added once + limiter.add_limit(5, 60) # add limits explicitly + limiter.parse_limits('5 per minute').parse_limits('5 in 60s', '5/60seconds') # add limits through text + + + # Test the limit without increasing the hits + for _ in range(100): + assert limiter.test_allow() + + # the limiter will allow 5 calls ... + for _ in range(5): + assert limiter.allow() + + # and reject the 6th + assert not limiter.allow() + + # It's possible to get statistics about the limiter and the corresponding windows + print(limiter.info()) + + +Documentation +^^^^^^^^^^^^^^^^^^ +.. autofunction:: HABApp.util.RateLimiter + + +.. autoclass:: HABApp.util.rate_limiter.limiter.Limiter + :members: + + Statistics ------------------------------ diff --git a/readme.md b/readme.md index 0662d034..50f941f8 100644 --- a/readme.md +++ b/readme.md @@ -127,7 +127,12 @@ MyOpenhabRule() ``` # Changelog -#### 23.11.1 (2023-11-23) +#### 23.12.0-DEV (2023-XX-XX) +- Added HABApp.util.RateLimiter +- Added CompressedMidnightRotatingFileHandler +- Updated dependencies + +#### 23.11.0 (2023-11-23) - Fix for very small float values (#425) - Fix for writing to persistence (#424) - Updated dependencies diff --git a/requirements_setup.txt b/requirements_setup.txt index 9251d495..8264b60c 100644 --- a/requirements_setup.txt +++ b/requirements_setup.txt @@ -1,4 +1,4 @@ -aiohttp == 3.9.0 +aiohttp == 3.9.1 pydantic == 2.5.2 msgspec == 0.18.4 pendulum == 2.1.2 diff --git a/run/conf/logging.yml b/run/conf/logging.yml index 6b8b363a..d4b0ce30 100644 --- a/run/conf/logging.yml +++ b/run/conf/logging.yml @@ -8,13 +8,15 @@ handlers: # There are several Handlers available: # - logging.handlers.RotatingFileHandler: # Will rotate when the file reaches a certain size (see python logging documentation for args) - # - HABApp.core.lib.handler.MidnightRotatingFileHandler: + # - HABApp.config.logging.handler.MidnightRotatingFileHandler: # Will wait until the file reaches a certain size and then will rotate on midnight + # - HABApp.config.logging.handler.CompressedMidnightRotatingFileHandler: + # Same as MidnightRotatingFileHandler but will rotate to a gzipped archive # - More handlers: # https://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler HABApp_default: - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.handler.MidnightRotatingFileHandler filename: 'HABApp.log' maxBytes: 1_048_576 backupCount: 3 @@ -23,7 +25,7 @@ handlers: level: DEBUG EventFile: - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.handler.CompressedMidnightRotatingFileHandler filename: 'events.log' maxBytes: 1_048_576 backupCount: 3 diff --git a/src/HABApp/__version__.py b/src/HABApp/__version__.py index 3bd0717c..b0cd1b76 100644 --- a/src/HABApp/__version__.py +++ b/src/HABApp/__version__.py @@ -10,4 +10,4 @@ # Development versions contain the DEV-COUNTER postfix: # - 23.09.0.DEV-1 -__version__ = '23.11.0' +__version__ = '23.12.0.DEV-1' diff --git a/src/HABApp/config/logging/__init__.py b/src/HABApp/config/logging/__init__.py index ac466ba4..b0c0e311 100644 --- a/src/HABApp/config/logging/__init__.py +++ b/src/HABApp/config/logging/__init__.py @@ -1,4 +1,4 @@ -from .handler import MidnightRotatingFileHandler +from .handler import MidnightRotatingFileHandler, CompressedMidnightRotatingFileHandler # isort: split diff --git a/src/HABApp/config/logging/default_logfile.py b/src/HABApp/config/logging/default_logfile.py index 67f58d71..3326232a 100644 --- a/src/HABApp/config/logging/default_logfile.py +++ b/src/HABApp/config/logging/default_logfile.py @@ -18,6 +18,8 @@ def get_default_logfile() -> str: # Will rotate when the file reaches a certain size (see python logging documentation for args) # - HABApp.config.logging.MidnightRotatingFileHandler: # Will wait until the file reaches a certain size and then will rotate on midnight + # - HABApp.config.logging.CompressedMidnightRotatingFileHandler: + # Same as MidnightRotatingFileHandler but will rotate to a gzipped archive # - More handlers: # https://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler diff --git a/src/HABApp/config/logging/handler.py b/src/HABApp/config/logging/handler.py index 95cb4a01..ad920d22 100644 --- a/src/HABApp/config/logging/handler.py +++ b/src/HABApp/config/logging/handler.py @@ -1,8 +1,14 @@ +import gzip +import shutil from datetime import date, datetime from logging.handlers import RotatingFileHandler +from pathlib import Path class MidnightRotatingFileHandler(RotatingFileHandler): + """A rotating file handler that checks once after midnight if the configured size has been exceeded and + then rotates the file + """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -14,3 +20,23 @@ def shouldRollover(self, record): return 0 self.last_check = date return super().shouldRollover(record) + + +class CompressedMidnightRotatingFileHandler(MidnightRotatingFileHandler): + """Same as ``MidnightRotatingFileHandler`` but rotates the file to a gzipped archive (``.gz``) + + """ + + def __init__(self, *args, **kwargs): + self.namer = self.compressed_namer + self.rotator = self.compressed_rotator + super().__init__(*args, **kwargs) + + def compressed_namer(self, default_name: str) -> str: + return default_name + ".gz" + + def compressed_rotator(self, source: str, dest: str): + src = Path(source) + with src.open('rb') as f_in, gzip.open(dest, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + src.unlink() diff --git a/src/HABApp/util/__init__.py b/src/HABApp/util/__init__.py index 2b09100d..8b56fd5a 100644 --- a/src/HABApp/util/__init__.py +++ b/src/HABApp/util/__init__.py @@ -2,6 +2,7 @@ from .statistics import Statistics from .listener_groups import EventListenerGroup from .fade import Fade +from .rate_limiter import RateLimiter from . import functions from . import multimode diff --git a/src/HABApp/util/rate_limiter/__init__.py b/src/HABApp/util/rate_limiter/__init__.py new file mode 100644 index 00000000..558dc8ea --- /dev/null +++ b/src/HABApp/util/rate_limiter/__init__.py @@ -0,0 +1 @@ +from .registry import RateLimiter diff --git a/src/HABApp/util/rate_limiter/limiter.py b/src/HABApp/util/rate_limiter/limiter.py new file mode 100644 index 00000000..0c096091 --- /dev/null +++ b/src/HABApp/util/rate_limiter/limiter.py @@ -0,0 +1,143 @@ +import re +from dataclasses import dataclass +from time import monotonic +from typing import Final, List, Tuple + +from .rate_limit import RateLimit, RateLimitInfo + + +LIMIT_REGEX = re.compile( + r""" + \s* ([1-9][0-9]*) + \s* (/|per|in) + \s* ([1-9][0-9]*)? + \s* (s|sec|second|m|min|minute|h|hour|day|month|year)s? + \s*""", + re.IGNORECASE | re.VERBOSE, +) + + +def parse_limit(text: str) -> Tuple[int, int]: + if not isinstance(text, str) or not (m := LIMIT_REGEX.fullmatch(text)): + msg = f'Invalid limit string: "{text:s}"' + raise ValueError(msg) + + count, per, factor, interval = m.groups() + + interval_secs = { + 's': 1, 'sec': 1, 'second': 1, 'm': 60, 'min': 60, 'minute': 60, 'hour': 3600, 'h': 3600, + 'day': 24 * 3600, 'month': 30 * 24 * 3600, 'year': 365 * 24 * 3600 + }[interval] + + return int(count), int(1 if factor is None else factor) * interval_secs + + +class Limiter: + def __init__(self, name: str): + self._name: Final = name + self._limits: Tuple[RateLimit, ...] = () + self._skipped = 0 + + def __repr__(self): + return f'<{self.__class__.__name__} {self._name:s}>' + + def add_limit(self, allowed: int, expiry: int) -> 'Limiter': + """Add a new rate limit + + :param allowed: How many hits are allowed + :param expiry: Interval in seconds + """ + if allowed <= 0 or not isinstance(allowed, int): + msg = f'Allowed must be an int >= 0, is {allowed} ({type(allowed)})' + raise ValueError(msg) + + if expiry <= 0 or not isinstance(expiry, int): + msg = f'Expire time must be an int >= 0, is {expiry} ({type(expiry)})' + raise ValueError(msg) + + for window in self._limits: + if window.allowed == allowed and window.expiry == expiry: + return self + + limit = RateLimit(allowed, expiry) + self._limits = tuple(sorted([*self._limits, limit], key=lambda x: x.expiry)) + return self + + def parse_limits(self, *text: str) -> 'Limiter': + """Add one or more limits in textual form, e.g. ``5 in 60s``, ``10 per hour`` or ``10/15 mins``. + If the limit does already exist it will not be added again. + + :param text: textual description of limit + """ + for limit in [parse_limit(t) for t in text]: + self.add_limit(*limit) + return self + + def allow(self) -> bool: + """Test the limit. + + :return: True if allowed, False if forbidden + """ + allow = True + clear_skipped = True + + if not self._limits: + msg = 'No limits defined!' + raise ValueError(msg) + + for limit in self._limits: + if not limit.allow(): + allow = False + # allow increments hits, if it's now 1 it was 0 before + if limit.hits != 1: + clear_skipped = False + + if clear_skipped: + self._skipped = 0 + + if not allow: + self._skipped += 1 + + return allow + + def test_allow(self) -> bool: + """Test the limit without hitting it. Calling this will not increase the hit counter. + + :return: True if allowed, False if forbidden + """ + allow = True + clear_skipped = True + + if not self._limits: + msg = 'No limits defined!' + raise ValueError(msg) + + for limit in self._limits: + if not limit.test_allow(): + allow = False + if limit.hits != 0: + clear_skipped = False + + if clear_skipped: + self._skipped = 0 + return allow + + def info(self) -> 'LimiterInfo': + """Get some info about the limiter and the defined windows + """ + now = monotonic() + remaining = max((w.stop for w in self._limits if w.hits), default=now) - now + if remaining <= 0: + remaining = 0 + + return LimiterInfo( + time_remaining=remaining, skipped=self._skipped, + limits=[limit.window_info() for limit in self._limits] + ) + + +@dataclass +class LimiterInfo: + time_remaining: float #: time remaining until skipped will reset + skipped: int #: how many entries were skipped + limits: List['RateLimitInfo'] # Info for every window diff --git a/src/HABApp/util/rate_limiter/rate_limit.py b/src/HABApp/util/rate_limiter/rate_limit.py new file mode 100644 index 00000000..500b559b --- /dev/null +++ b/src/HABApp/util/rate_limiter/rate_limit.py @@ -0,0 +1,73 @@ +from dataclasses import dataclass +from time import monotonic +from typing import Final + + +@dataclass +class RateLimitInfo: + time_remaining: float #: Time remaining until this window will reset + hits: int #: Hits + skips: int #: Skips + limit: int #: Boundary + + @property + def hits_remaining(self) -> int: + return self.limit - self.hits + + +class RateLimit: + def __init__(self, allowed: int, expiry: int): + super().__init__() + assert allowed > 0, allowed + assert expiry > 0, expiry + + self.expiry: Final = expiry + self.allowed: Final = allowed + + self.start: float = -1.0 + self.stop: float = -1.0 + self.hits: int = 0 + self.skips: int = 0 + + def __repr__(self): + return (f'<{self.__class__.__name__} hits={self.hits:d}/{self.allowed:d} ' + f'expiry={self.expiry:d}s window={self.stop - self.start:.0f}s>') + + def allow(self) -> bool: + now = monotonic() + + if self.stop < now: + self.hits = 0 + self.skips = 0 + self.start = now + self.stop = now + self.expiry + + self.hits += 1 + if self.hits <= self.allowed: + return True + + self.skips += 1 + self.hits = self.allowed + self.stop = now + self.expiry + return False + + def test_allow(self) -> bool: + now = monotonic() + + if self.hits and self.stop < now: + self.hits = 0 + self.skips = 0 + + return self.hits < self.allowed + + def window_info(self) -> RateLimitInfo: + if self.hits <= 0: + remaining = self.expiry + else: + remaining = self.stop - monotonic() + if remaining <= 0: + remaining = 0 + + return RateLimitInfo( + time_remaining=remaining, hits=self.hits, skips=self.skips, limit=self.allowed + ) diff --git a/src/HABApp/util/rate_limiter/registry.py b/src/HABApp/util/rate_limiter/registry.py new file mode 100644 index 00000000..048c7e40 --- /dev/null +++ b/src/HABApp/util/rate_limiter/registry.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from threading import Lock + +from .limiter import Limiter + + +LOCK = Lock() + +_LIMITERS: dict[str, Limiter] = {} + + +def RateLimiter(name: str) -> Limiter: + """Create a new rate limiter or return an already existing one with a given name. + + :param name: case insensitive name of limiter + :return: Rate limiter object + """ + + key = name.lower() + + with LOCK: + if (obj := _LIMITERS.get(key)) is None: + _LIMITERS[key] = obj = Limiter(name) + + return obj diff --git a/tests/test_utils/test_rate_limiter.py b/tests/test_utils/test_rate_limiter.py new file mode 100644 index 00000000..820ec820 --- /dev/null +++ b/tests/test_utils/test_rate_limiter.py @@ -0,0 +1,166 @@ +import pytest + +import HABApp.util.rate_limiter.limiter as limiter_module +import HABApp.util.rate_limiter.rate_limit as rate_limit_module +import HABApp.util.rate_limiter.registry as registry_module +from HABApp.util.rate_limiter.limiter import Limiter, RateLimit, parse_limit + + +@pytest.mark.parametrize( + 'unit,factor', ( + ('s', 1), ('sec', 1), ('second', 1), + ('m', 60), ('min', 60), ('minute', 60), + ('h', 3600), ('hour', 3600), + ('day', 24 * 3600), ('month', 30 * 24 * 3600), ('year', 365 * 24 * 3600) + ) +) +def test_parse(unit: str, factor: int): + assert parse_limit(f' 1 per {unit} ') == (1, factor) + assert parse_limit(f' 1 / {unit} ') == (1, factor) + assert parse_limit(f'3 per {unit}') == (3, factor) + assert parse_limit(f'3 in {unit}') == (3, factor) + + for ctr in (1, 12, 375, 5533): + assert parse_limit(f'{ctr:d} in 5{unit}') == (ctr, 5 * factor) + assert parse_limit(f'{ctr:d} in 5{unit}s') == (ctr, 5 * factor) + + with pytest.raises(ValueError) as e: + parse_limit('asdf') + assert str(e.value) == 'Invalid limit string: "asdf"' + + +def test_window(monkeypatch): + time = 0 + monkeypatch.setattr(rate_limit_module, 'monotonic', lambda: time) + + limit = RateLimit(5, 3) + assert str(limit) == '' + assert limit.test_allow() + + assert limit.allow() + assert str(limit) == '' + + for _ in range(4): + assert limit.allow() + + assert str(limit) == '' + + # Limit is full, stop gets moved further + time = 1 + assert not limit.allow() + assert str(limit) == '' + + # move out of interval + time = 4.1 + assert limit.allow() + assert limit.hits == 1 + assert str(limit) == '' + + +def test_window_test_allow(monkeypatch): + time = 0 + monkeypatch.setattr(rate_limit_module, 'monotonic', lambda: time) + + limit = RateLimit(5, 3) + limit.hits = 5 + limit.stop = 2.99999 + assert not limit.test_allow() + + # expiry when out of window + time = 3 + assert limit.test_allow() + assert not limit.hits + + +def test_limiter_add(monkeypatch): + limiter = Limiter('test') + limiter.add_limit(3, 5).add_limit(3, 5).parse_limits('3 in 5s') + assert len(limiter._limits) == 1 + + +def test_limiter_info(monkeypatch): + time = 0 + monkeypatch.setattr(rate_limit_module, 'monotonic', lambda: time) + monkeypatch.setattr(limiter_module, 'monotonic', lambda: time) + + limiter = Limiter('test') + + info = limiter.info() + assert info.time_remaining == 0 + assert info.skipped == 0 + + with pytest.raises(ValueError): + limiter.allow() + + with pytest.raises(ValueError): + limiter.test_allow() + + limiter.add_limit(3, 3) + + info = limiter.info() + assert info.time_remaining == 0 + assert info.skipped == 0 + + w_info = info.limits[0] + assert w_info.limit == 3 + assert w_info.skips == 0 + assert w_info.time_remaining == 3 + assert w_info.hits == 0 + + limiter.allow() + time = 2 + limiter.allow() + + info = limiter.info() + assert info.time_remaining == 1 + assert info.skipped == 0 + + w_info = info.limits[0] + assert w_info.limit == 3 + assert w_info.skips == 0 + assert w_info.time_remaining == 1 + assert w_info.hits == 2 + + # add a longer limiter - this one should now define the time_remaining + limiter.add_limit(4, 5) + limiter.allow() + + info = limiter.info() + assert info.time_remaining == 5 + assert info.skipped == 0 + + w_info = info.limits[0] + assert w_info.limit == 3 + assert w_info.skips == 0 + assert w_info.time_remaining == 1 + assert w_info.hits == 3 + + w_info = info.limits[1] + assert w_info.limit == 4 + assert w_info.skips == 0 + assert w_info.time_remaining == 5 + assert w_info.hits == 1 + + time += 5.0001 + + info = limiter.info() + assert info.time_remaining == 0 + + w_info = info.limits[0] + assert w_info.limit == 3 + assert w_info.skips == 0 + assert w_info.time_remaining == 0 + assert w_info.hits == 3 + + w_info = info.limits[1] + assert w_info.limit == 4 + assert w_info.skips == 0 + assert w_info.time_remaining == 0 + assert w_info.hits == 1 + + +def test_registry(monkeypatch): + monkeypatch.setattr(registry_module, '_LIMITERS', {}) + + obj = registry_module.RateLimiter('test') + assert obj is registry_module.RateLimiter('TEST')