diff --git a/Dockerfile b/Dockerfile index bdc42242..4da02129 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,8 +7,7 @@ RUN apk add --no-cache \ # Support for Timezones tzdata \ # ujson won't compile without these libs - musl-dev \ - gcc + g++ # Always use latest versions RUN mkdir -p /usr/src/app diff --git a/HABApp/__version__.py b/HABApp/__version__.py index f9c66296..2c69a781 100644 --- a/HABApp/__version__.py +++ b/HABApp/__version__.py @@ -1 +1 @@ -__VERSION__ = '0.12.3' +__VERSION__ = '0.12.4' diff --git a/HABApp/config/config_loader.py b/HABApp/config/config_loader.py index 70beeaf9..811e8ed9 100644 --- a/HABApp/config/config_loader.py +++ b/HABApp/config/config_loader.py @@ -8,7 +8,6 @@ import ruamel.yaml from HABApp.__version__ import __VERSION__ -from HABApp.runtime import FileEventTarget from . import CONFIG from .default_logfile import get_default_logfile @@ -31,7 +30,7 @@ class InvalidConfigException(Exception): pass -class HABAppConfigLoader(FileEventTarget): +class HABAppConfigLoader: def __init__(self, config_folder: Path): @@ -48,25 +47,19 @@ def __init__(self, config_folder: Path): self.first_start = True try: # try load logging config first. If we use abs path we can log errors when loading config.yml - self.on_file_added(self.file_conf_logging) - self.on_file_added(self.file_conf_habapp) + self.on_file_event(self.file_conf_logging) + self.on_file_event(self.file_conf_habapp) except AbsolutePathExpected: - self.on_file_added(self.file_conf_habapp) - self.on_file_added(self.file_conf_logging) + self.on_file_event(self.file_conf_habapp) + self.on_file_event(self.file_conf_logging) self.first_start = False - def on_file_added(self, path: Path): - self.on_file_changed(path) - - def on_file_changed(self, path: Path): + def on_file_event(self, path: Path): if path.name == 'config.yml': self.load_cfg() if path.name == 'logging.yml': self.load_log() - def on_file_removed(self, path: Path): - pass - def __check_create_logging(self): if self.file_conf_logging.is_file(): return None @@ -90,7 +83,7 @@ def load_cfg(self): return None def load_log(self): - # File has to exist - check because we also get FileDelete events + # config gets created on startup - if it gets deleted we do nothing here if not self.file_conf_logging.is_file(): return None diff --git a/HABApp/core/EventBus.py b/HABApp/core/EventBus.py index 782c2178..6d130d79 100644 --- a/HABApp/core/EventBus.py +++ b/HABApp/core/EventBus.py @@ -2,7 +2,7 @@ import threading import typing -from HABApp.util import log_exception +from HABApp.core.wrapper import log_exception from . import EventBusListener from .events import ComplexEventValue diff --git a/HABApp/core/__init__.py b/HABApp/core/__init__.py index 279e0cad..34fc1733 100644 --- a/HABApp/core/__init__.py +++ b/HABApp/core/__init__.py @@ -1,4 +1,5 @@ from . import const +from . import wrapper from .wrappedfunction import WrappedFunction diff --git a/HABApp/core/const/__init__.py b/HABApp/core/const/__init__.py index 497728bc..a10374fe 100644 --- a/HABApp/core/const/__init__.py +++ b/HABApp/core/const/__init__.py @@ -1,3 +1,4 @@ -from .const import MISSING +from . import json from . import topics +from .const import MISSING from .loop import loop diff --git a/HABApp/core/const/json.py b/HABApp/core/const/json.py new file mode 100644 index 00000000..efeedf03 --- /dev/null +++ b/HABApp/core/const/json.py @@ -0,0 +1,8 @@ +try: + import ujson + load_json = ujson.loads + dump_json = ujson.dumps +except ImportError: + import json + load_json = json.loads + dump_json = json.dumps diff --git a/HABApp/core/items/item_times.py b/HABApp/core/items/item_times.py index d66c7ac4..cbfaf85e 100644 --- a/HABApp/core/items/item_times.py +++ b/HABApp/core/items/item_times.py @@ -3,7 +3,7 @@ import typing import HABApp -from HABApp.util.wrapper import log_exception +from HABApp.core.wrapper import log_exception from ..const import loop from ..events import ItemNoChangeEvent, ItemNoUpdateEvent @@ -11,7 +11,7 @@ class BaseWatch: EVENT: typing.Union[typing.Type[ItemNoUpdateEvent], typing.Type[ItemNoChangeEvent]] - def __init__(self, name: str, secs: int): + def __init__(self, name: str, secs: typing.Union[int, float]): self._secs: typing.Union[int, float] = secs self._name: str = name self._task: typing.Optional[asyncio.Task] = None @@ -75,7 +75,7 @@ def set(self, dt: datetime.datetime, events=True): asyncio.run_coroutine_threadsafe(self.schedule_events(), loop) return None - def add_watch(self, secs: int) -> BaseWatch: + def add_watch(self, secs: typing.Union[int, float]) -> BaseWatch: assert secs > 0, secs # don't add the watch two times diff --git a/HABApp/core/wrappedfunction.py b/HABApp/core/wrappedfunction.py index 668d0de0..4abd969f 100644 --- a/HABApp/core/wrappedfunction.py +++ b/HABApp/core/wrappedfunction.py @@ -1,8 +1,16 @@ import asyncio import concurrent.futures +import io import logging import time import traceback +from cProfile import Profile +from pstats import Stats +try: + from pstats import SortKey + STAT_SORT_KEY = SortKey.CUMULATIVE +except ImportError: + STAT_SORT_KEY = 'cumulative', 'cumtime' import HABApp @@ -27,7 +35,6 @@ def __init__(self, func, logger=None, warn_too_long=True, name=None): # Allow custom logger self.log = default_logger if logger: - assert isinstance(logger, logging.getLoggerClass()) self.log = logger self.__warn_too_long = warn_too_long @@ -80,13 +87,28 @@ def __run(self, *args, **kwargs): self.log.warning(f'Starting of {self.name} took too long: {__start - self.__time_submitted:.2f}s. ' f'Maybe there are not enough threads?') + # start profiler + pr = Profile() + pr.enable() + # Execute the function try: self._func(*args, **kwargs) except Exception as e: self.__format_traceback(e, *args, **kwargs) + # disable profiler + pr.disable() + # log warning if execution takes too long __dur = time.time() - __start if self.__warn_too_long and __dur > 0.8: self.log.warning(f'Execution of {self.name} took too long: {__dur:.2f}s') + + s = io.StringIO() + ps = Stats(pr, stream=s).sort_stats(STAT_SORT_KEY) + ps.print_stats(0.1) # limit to output to 10% of the lines + + for line in s.getvalue().splitlines()[4:]: # skip the amount of calls and "Ordered by:" + if line: + self.log.warning(line) diff --git a/HABApp/util/wrapper.py b/HABApp/core/wrapper.py similarity index 80% rename from HABApp/util/wrapper.py rename to HABApp/core/wrapper.py index 955a2204..b1b6d4ce 100644 --- a/HABApp/util/wrapper.py +++ b/HABApp/core/wrapper.py @@ -1,115 +1,120 @@ -import functools -import logging -import traceback -import asyncio -import sys - -import HABApp - -log = logging.getLogger('HABApp') - - -def __process_exception(func, e: Exception, do_print=False): - lines = traceback.format_exc().splitlines() - del lines[1:3] # Remove entries which point to this wrapper - - # log exception, since it is unexpected we push it to stdout, too - if do_print: - print(f'Error {e} in {func.__name__}:') - log.error(f'Error {e} in {func.__name__}:') - for line in lines: - if do_print: - print(line) - log.error(line) - - # send Error to internal event bus so we can reprocess it and notify the user - HABApp.core.EventBus.post_event( - 'HABApp.Errors', HABApp.core.events.habapp_events.HABAppError( - func_name=func.__name__, exception=e, traceback='\n'.join(lines) - ) - ) - - -def log_exception(func): - # return async wrapper - if asyncio.iscoroutine(func) or asyncio.iscoroutinefunction(func): - @functools.wraps(func) - async def a(*args, **kwargs): - try: - return await func(*args, **kwargs) - except asyncio.CancelledError: - pass - except Exception as e: - __process_exception(func, e, do_print=True) - # re raise exception, since this is something we didn't anticipate - raise - - return a - - @functools.wraps(func) - def f(*args, **kwargs): - try: - return func(*args, **kwargs) - except Exception as e: - __process_exception(func, e, do_print=True) - # re raise exception, since this is something we didn't anticipate - raise - - return f - - -def ignore_exception(func): - # return async wrapper - if asyncio.iscoroutine(func) or asyncio.iscoroutinefunction(func): - @functools.wraps(func) - async def a(*args, **kwargs): - try: - return await func(*args, **kwargs) - except asyncio.CancelledError: - pass - except Exception as e: - __process_exception(func, e) - return None - - return a - - @functools.wraps(func) - def f(*args, **kwargs): - try: - return func(*args, **kwargs) - except Exception as e: - __process_exception(func, e) - return None - return f - - -class ExceptionToHABApp: - def __init__(self, logger=None, ignore_exception=True): - self.log = logger - self.ignore_exception = ignore_exception - - def __enter__(self): - pass - - def __exit__(self, exc_type, exc_val, exc_tb): - tb = traceback.format_exc() - - # try to get the parent function name - try: - f_name = sys._getframe().f_back.f_code.co_name - except Exception: - f_name = 'Exception while getting the function name!' - - # log error - if self.log is not None: - self.log.error(f'Error {exc_val} in {f_name}:') - for l in tb.splitlines(): - self.log.error(l) - - # send Error to internal event bus so we can reprocess it and notify the user - HABApp.core.EventBus.post_event( - 'HABApp.Errors', HABApp.core.events.habapp_events.HABAppError( - func_name=f_name, exception=exc_val, traceback=tb - ) - ) - return self.ignore_exception +import asyncio +import functools +import logging +import sys +import traceback +import typing +from logging import Logger + +import HABApp + +log = logging.getLogger('HABApp') + + +def __process_exception(func, e: Exception, do_print=False): + lines = traceback.format_exc().splitlines() + del lines[1:3] # Remove entries which point to this wrapper + + # log exception, since it is unexpected we push it to stdout, too + if do_print: + print(f'Error {e} in {func.__name__}:') + log.error(f'Error {e} in {func.__name__}:') + for line in lines: + if do_print: + print(line) + log.error(line) + + # send Error to internal event bus so we can reprocess it and notify the user + HABApp.core.EventBus.post_event( + HABApp.core.const.topics.ERRORS, HABApp.core.events.habapp_events.HABAppError( + func_name=func.__name__, exception=e, traceback='\n'.join(lines) + ) + ) + + +def log_exception(func): + # return async wrapper + if asyncio.iscoroutine(func) or asyncio.iscoroutinefunction(func): + @functools.wraps(func) + async def a(*args, **kwargs): + try: + return await func(*args, **kwargs) + except asyncio.CancelledError: + pass + except Exception as e: + __process_exception(func, e, do_print=True) + # re raise exception, since this is something we didn't anticipate + raise + + return a + + @functools.wraps(func) + def f(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + __process_exception(func, e, do_print=True) + # re raise exception, since this is something we didn't anticipate + raise + + return f + + +def ignore_exception(func): + # return async wrapper + if asyncio.iscoroutine(func) or asyncio.iscoroutinefunction(func): + @functools.wraps(func) + async def a(*args, **kwargs): + try: + return await func(*args, **kwargs) + except asyncio.CancelledError: + pass + except Exception as e: + __process_exception(func, e) + return None + + return a + + @functools.wraps(func) + def f(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + __process_exception(func, e) + return None + return f + + +class ExceptionToHABApp: + def __init__(self, logger: typing.Optional[Logger] = None, log_level: int = logging.ERROR, + ignore_exception: bool = True): + self.log: typing.Optional[Logger] = logger + self.log_level = log_level + self.ignore_exception: bool = ignore_exception + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + tb = traceback.format_exc() + + # try to get the parent function name + try: + f_name = sys._getframe().f_back.f_code.co_name + except Exception: + f_name = 'Exception while getting the function name!' + + # log error + if self.log is not None: + self.log.log(self.log_level, f'Error {exc_val} in {f_name}:') + for l in tb.splitlines(): + self.log.log(self.log_level, l) + + # send Error to internal event bus so we can reprocess it and notify the user + HABApp.core.EventBus.post_event( + HABApp.core.const.topics.WARNINGS if self.log_level == logging.WARNING else HABApp.core.const.topics.ERRORS, + HABApp.core.events.habapp_events.HABAppError( + func_name=f_name, exception=exc_val, traceback=tb + ) + ) + return self.ignore_exception diff --git a/HABApp/mqtt/mqtt_connection.py b/HABApp/mqtt/mqtt_connection.py index 242c3440..45897261 100644 --- a/HABApp/mqtt/mqtt_connection.py +++ b/HABApp/mqtt/mqtt_connection.py @@ -1,14 +1,14 @@ import logging import typing -import ujson import paho.mqtt.client as mqtt import HABApp -from HABApp.util import log_exception +from HABApp.core.wrapper import log_exception from HABApp.runtime.shutdown_helper import ShutdownHelper from ..config import Mqtt as MqttConfig +from ..core.const.json import load_json from .events import MqttValueUpdateEvent, MqttValueChangeEvent @@ -138,7 +138,7 @@ def process_msg(self, client, userdata, message: mqtt.MQTTMessage): # load json dict and list if payload.startswith('{') and payload.endswith('}') or payload.startswith('[') and payload.endswith(']'): try: - payload = ujson.loads(payload) + payload = load_json(payload) except ValueError: pass else: diff --git a/HABApp/mqtt/mqtt_interface.py b/HABApp/mqtt/mqtt_interface.py index b3394066..3e0e6da3 100644 --- a/HABApp/mqtt/mqtt_interface.py +++ b/HABApp/mqtt/mqtt_interface.py @@ -1,10 +1,10 @@ import typing import paho.mqtt.client as mqtt -import ujson from .mqtt_connection import MqttConnection, log from ..config import Mqtt as MqttConfig +from ..core.const.json import dump_json class MqttInterface: @@ -51,7 +51,7 @@ def publish(self, topic: str, payload: typing.Any, qos: int = None, retain: bool # convert these to string if isinstance(payload, (dict, list)): - payload = ujson.dumps(payload) + payload = dump_json(payload) info = self.__connection.client.publish(topic, payload, qos, retain) if info.rc != mqtt.MQTT_ERR_SUCCESS: diff --git a/HABApp/openhab/events/item_events.py b/HABApp/openhab/events/item_events.py index feae74c7..58fb2064 100644 --- a/HABApp/openhab/events/item_events.py +++ b/HABApp/openhab/events/item_events.py @@ -60,7 +60,7 @@ def __repr__(self): class ItemAddedEvent(OpenhabEvent): - def __init__(self, name: str = '', type: str = None): + def __init__(self, name: str, type: str): super().__init__() self.name: str = name @@ -78,7 +78,7 @@ def __repr__(self): class ItemUpdatedEvent(OpenhabEvent): - def __init__(self, name: str = '', type: str = None): + def __init__(self, name: str, type: str): super().__init__() self.name: str = name diff --git a/HABApp/openhab/http_connection.py b/HABApp/openhab/http_connection.py index 75fbd599..92610709 100644 --- a/HABApp/openhab/http_connection.py +++ b/HABApp/openhab/http_connection.py @@ -5,7 +5,6 @@ import aiohttp import datetime -import ujson from aiohttp.client import ClientResponse from aiohttp_sse_client import client as sse_client @@ -13,6 +12,7 @@ import HABApp.core import HABApp.openhab.events from ..config import Openhab as OpenhabConfig +from ..core.const.json import load_json, dump_json log = logging.getLogger('HABApp.openhab.connection') log_events = logging.getLogger('HABApp.EventBus.openhab') @@ -170,7 +170,7 @@ async def try_connect(self): self.__session = aiohttp.ClientSession( timeout=aiohttp.ClientTimeout(total=99999999999999999), - json_serialize=ujson.dumps, + json_serialize=dump_json, auth=auth ) @@ -217,7 +217,7 @@ async def async_process_sse_events(self): ) as event_source: async for event in event_source: try: - event = ujson.loads(event.data) + event = load_json(event.data) except ValueError: continue except TypeError: @@ -289,7 +289,7 @@ async def async_get_items(self) -> typing.Optional[list]: ) try: resp = await self._check_http_response(fut) - return ujson.loads(await resp.text(encoding='utf-8')) + return load_json(await resp.text(encoding='utf-8')) except Exception as e: # sometimes uuid already works but items not - so we ignore these errors here, too if not isinstance(e, (OpenhabDisconnectedError, OpenhabNotReadyYet)): @@ -301,7 +301,7 @@ async def async_get_things(self) -> typing.Optional[typing.List[dict]]: fut = self.__session.get(self.__get_openhab_url('rest/things')) try: resp = await self._check_http_response(fut) - return ujson.loads(await resp.text(encoding='utf-8')) + return load_json(await resp.text(encoding='utf-8')) except Exception as e: # sometimes uuid and items already works but things not - so we ignore these errors here, too if not isinstance(e, (OpenhabDisconnectedError, OpenhabNotReadyYet)): diff --git a/HABApp/openhab/map_events.py b/HABApp/openhab/map_events.py index d4547776..81aa5da9 100644 --- a/HABApp/openhab/map_events.py +++ b/HABApp/openhab/map_events.py @@ -1,5 +1,5 @@ import typing -import ujson +from ..core.const.json import load_json from .events import OpenhabEvent, \ ItemStateEvent, ItemStateChangedEvent, ItemCommandEvent, ItemAddedEvent, \ @@ -31,10 +31,10 @@ def get_event(_in_dict : dict) -> OpenhabEvent: p_str: str = _in_dict['payload'] if '"NONE"' in p_str: p_str = p_str.replace('"NONE"', 'null') - payload = ujson.loads(p_str) + payload = load_json(p_str) # Find event from implemented events try: - return __event_lookup[event_type]().from_dict(topic, payload) + return __event_lookup[event_type].from_dict(topic, payload) except KeyError: raise ValueError(f'Unknown Event: {event_type:s} for {_in_dict}') diff --git a/HABApp/openhab/oh_connection.py b/HABApp/openhab/oh_connection.py index 1681c374..7f276913 100644 --- a/HABApp/openhab/oh_connection.py +++ b/HABApp/openhab/oh_connection.py @@ -6,7 +6,7 @@ import HABApp.core import HABApp.openhab.events from HABApp.openhab.map_events import get_event -from HABApp.util import log_exception, ignore_exception +from HABApp.core.wrapper import log_exception, ignore_exception from .http_connection import HttpConnection, HttpConnectionEventHandler from .oh_interface import get_openhab_interface from ..config import Openhab as OpenhabConfig diff --git a/HABApp/openhab/oh_interface.py b/HABApp/openhab/oh_interface.py index 2dec0672..ea3efdc5 100644 --- a/HABApp/openhab/oh_interface.py +++ b/HABApp/openhab/oh_interface.py @@ -9,7 +9,7 @@ import HABApp.openhab.events from HABApp.core.const import loop from HABApp.core.items.base_valueitem import BaseValueItem -from HABApp.util import log_exception +from HABApp.core.wrapper import log_exception from . import definitions from .http_connection import HttpConnection diff --git a/HABApp/rule/interfaces/http.py b/HABApp/rule/interfaces/http.py index 45507950..e404c32d 100644 --- a/HABApp/rule/interfaces/http.py +++ b/HABApp/rule/interfaces/http.py @@ -1,7 +1,9 @@ -import aiohttp -import ujson from typing import Any, Optional, Mapping +import aiohttp + +from HABApp.core.const.json import dump_json + class AsyncHttpConnection: @@ -9,7 +11,7 @@ def __init__(self): self.__client: aiohttp.ClientSession = None async def create_client(self, loop): - self.__client = aiohttp.ClientSession(json_serialize=ujson.dumps, loop=loop) + self.__client = aiohttp.ClientSession(json_serialize=dump_json, loop=loop) def get(self, url: str, params: Optional[Mapping[str, str]] = None, **kwargs: Any)\ -> aiohttp.client._RequestContextManager: diff --git a/HABApp/rule/rule.py b/HABApp/rule/rule.py index 08a8a32c..8a23352b 100644 --- a/HABApp/rule/rule.py +++ b/HABApp/rule/rule.py @@ -17,6 +17,7 @@ from .interfaces import async_subprocess_exec from .scheduler import ReoccurringScheduledCallback, OneTimeCallback, DayOfWeekScheduledCallback, \ TYPING_DATE_TIME, SunScheduledCallback +from .scheduler.base import ScheduledCallbackBase as _ScheduledCallbackBase log = logging.getLogger('HABApp.Rule') @@ -60,7 +61,7 @@ def __init__(self): self.__rule_file: HABApp.rule_manager.RuleFile = __rule_file__ self.__event_listener: typing.List[HABApp.core.EventBusListener] = [] - self.__future_events: typing.List[OneTimeCallback] = [] + self.__future_events: typing.List[_ScheduledCallbackBase] = [] self.__unload_functions: typing.List[typing.Callable[[], None]] = [] self.__cancel_objs: weakref.WeakSet = weakref.WeakSet() @@ -78,13 +79,15 @@ def __init__(self): self.oh: HABApp.openhab.OpenhabInterface = HABApp.openhab.get_openhab_interface() if not test else None self.openhab: HABApp.openhab.OpenhabInterface = self.oh - @HABApp.util.log_exception + @HABApp.core.wrapper.log_exception def __cleanup_objs(self): while self.__cancel_objs: - obj = self.__cancel_objs.pop() - obj.cancel() + # we log each error as warning + with HABApp.core.wrapper.ExceptionToHABApp(log, logging.WARNING): + obj = self.__cancel_objs.pop() + obj.cancel() - @HABApp.util.log_exception + @HABApp.core.wrapper.log_exception def __cleanup_rule(self): # Important: set the dicts to None so we don't schedule a future event during _cleanup. # If dict is set to None we will crash instead but it is no problem because everything gets unloaded anyhow @@ -475,7 +478,7 @@ def get_rule_parameter(self, file_name: str, *keys, default_value='ToDo'): def __get_rule_name(self, callback): return f'{self.rule_name}.{callback.__name__}' if self.rule_name else None - @HABApp.util.log_exception + @HABApp.core.wrapper.log_exception def _check_rule(self): # Check if items do exists @@ -494,7 +497,7 @@ def _check_rule(self): f'self.listen_event in "{self.rule_name}" may not work as intended.') - @HABApp.util.log_exception + @HABApp.core.wrapper.log_exception def _process_events(self, now): # sheduled events @@ -510,7 +513,7 @@ def _process_events(self, now): self.__future_events = [k for k in self.__future_events if not k.is_finished] return None - @HABApp.util.log_exception + @HABApp.core.wrapper.log_exception def _unload(self): # unload all functions @@ -534,7 +537,7 @@ def _unload(self): log.error(line) -@HABApp.util.log_exception +@HABApp.core.wrapper.log_exception def get_parent_rule() -> Rule: depth = 1 while True: diff --git a/HABApp/rule_manager/rule_manager.py b/HABApp/rule_manager/rule_manager.py index baf27e87..ff2700fc 100644 --- a/HABApp/rule_manager/rule_manager.py +++ b/HABApp/rule_manager/rule_manager.py @@ -10,7 +10,7 @@ from pytz import utc import HABApp -from HABApp.util import log_exception +from HABApp.core.wrapper import log_exception from .rule_file import RuleFile from HABApp.core.const.topics import RULES as TOPIC_RULES diff --git a/HABApp/runtime/__init__.py b/HABApp/runtime/__init__.py index 65cec9d1..529a5895 100644 --- a/HABApp/runtime/__init__.py +++ b/HABApp/runtime/__init__.py @@ -1,3 +1,3 @@ from .shutdown_helper import ShutdownHelper -from .folder_watcher import FileEventTarget, FolderWatcher +from .folder_watcher import SimpleAsyncEventHandler, FolderWatcher from .runtime import Runtime \ No newline at end of file diff --git a/HABApp/runtime/folder_watcher/__init__.py b/HABApp/runtime/folder_watcher/__init__.py index e18cc9ba..404e9cdb 100644 --- a/HABApp/runtime/folder_watcher/__init__.py +++ b/HABApp/runtime/folder_watcher/__init__.py @@ -1,2 +1,2 @@ -from .simplefileevent import FileEventTarget +from .simpleasyncfileevent import SimpleAsyncEventHandler from .folder_watcher import FolderWatcher \ No newline at end of file diff --git a/HABApp/runtime/folder_watcher/folder_watcher.py b/HABApp/runtime/folder_watcher/folder_watcher.py index 7c141bb7..fd0b2762 100644 --- a/HABApp/runtime/folder_watcher/folder_watcher.py +++ b/HABApp/runtime/folder_watcher/folder_watcher.py @@ -4,14 +4,14 @@ from watchdog.observers import Observer from .habappfileevent import FileEventToHABAppEvent -from .simplefileevent import SimpleFileEventHandler +from .simpleasyncfileevent import SimpleAsyncEventHandler class FolderWatcher: def __init__(self): self.__observer = Observer() - self.__handlers: typing.Dict[str, typing.Union[SimpleFileEventHandler, FileEventToHABAppEvent]] = {} + self.__handlers: typing.Dict[str, typing.Union[SimpleAsyncEventHandler, FileEventToHABAppEvent]] = {} self.__watches = {} self.__started = False @@ -31,16 +31,16 @@ def start(self, shutdown_helper): shutdown_helper.register_func(self.__observer.join, last=True) return None - def watch_folder(self, folder: pathlib.Path, file_ending: str, event_target, - watch_subfolders = False, worker_factory=None) -> SimpleFileEventHandler: + def watch_folder(self, folder: pathlib.Path, file_ending: str, target_func, + watch_subfolders = False, worker_factory=None) -> SimpleAsyncEventHandler: assert isinstance(folder, pathlib.Path), type(folder) assert folder.is_dir(), folder folder_str = str(folder) assert folder_str not in self.__watches, folder_str - self.__handlers[folder_str] = handler = SimpleFileEventHandler( - event_target=event_target, file_ending=file_ending, worker_factory=worker_factory + self.__handlers[folder_str] = handler = SimpleAsyncEventHandler( + target_func=target_func, file_ending=file_ending, worker_factory=worker_factory ) self.__watches[folder_str] = self.__observer.schedule(handler, path=folder_str, recursive=watch_subfolders) return handler @@ -67,7 +67,7 @@ def unwatch_folder(self, folder): self.__handlers.pop(folder) self.__observer.unschedule(self.__watches.pop(folder)) - def get_handler(self, folder) -> typing.Union[SimpleFileEventHandler, FileEventToHABAppEvent]: + def get_handler(self, folder) -> typing.Union[SimpleAsyncEventHandler, FileEventToHABAppEvent]: if isinstance(folder, pathlib.Path): folder = str(folder) assert isinstance(folder, str), type(folder) diff --git a/HABApp/runtime/folder_watcher/habappfileevent.py b/HABApp/runtime/folder_watcher/habappfileevent.py index ddbf86ce..6442e553 100644 --- a/HABApp/runtime/folder_watcher/habappfileevent.py +++ b/HABApp/runtime/folder_watcher/habappfileevent.py @@ -1,66 +1,33 @@ import time from pathlib import Path -from watchdog.events import FileSystemEventHandler, FileMovedEvent, FileCreatedEvent, \ - FileDeletedEvent, FileModifiedEvent - import HABApp +from .simpleasyncfileevent import SimpleAsyncEventHandler -class FileEventToHABAppEvent(FileSystemEventHandler): +class FileEventToHABAppEvent(SimpleAsyncEventHandler): def __init__(self, folder: Path, habapp_topic: str, file_ending: str, recursive=False): assert isinstance(folder, Path), type(folder) assert isinstance(file_ending, str), type(file_ending) assert isinstance(habapp_topic, str), type(habapp_topic) assert isinstance(recursive, bool), type(recursive) + super().__init__(self.create_habapp_event, file_ending) + self.folder: Path = folder self.habapp_topic: str = habapp_topic - self.file_ending: str = file_ending self.recursive: bool = recursive - def send_habapp_event(self, path: str, event): + def create_habapp_event(self, path: Path): + if path.is_file(): + event = HABApp.core.events.habapp_events.RequestFileLoadEvent + else: + event = HABApp.core.events.habapp_events.RequestFileUnloadEvent + HABApp.core.EventBus.post_event( - self.habapp_topic, event.from_path(self.folder, Path(path)) + self.habapp_topic, event.from_path(self.folder, path) ) - def on_deleted(self, event): - if not isinstance(event, FileDeletedEvent): - return None - - if not event.src_path.endswith(self.file_ending): - return None - - self.send_habapp_event(event.src_path, HABApp.core.events.habapp_events.RequestFileUnloadEvent) - - def on_created(self, event): - if not isinstance(event, FileCreatedEvent): - return None - - if not event.src_path.endswith(self.file_ending): - return None - - self.send_habapp_event(event.src_path, HABApp.core.events.habapp_events.RequestFileLoadEvent) - - def on_moved(self, event): - if not isinstance(event, FileMovedEvent): - return None - - if event.src_path.endswith(self.file_ending): - self.send_habapp_event(event.src_path, HABApp.core.events.habapp_events.RequestFileUnloadEvent) - - if event.dest_path.endswith(self.file_ending): - self.send_habapp_event(event.dest_path, HABApp.core.events.habapp_events.RequestFileLoadEvent) - - def on_modified(self, event): - if not isinstance(event, FileModifiedEvent): - return None - - if not event.src_path.endswith(self.file_ending): - return None - - self.send_habapp_event(event.src_path, HABApp.core.events.habapp_events.RequestFileLoadEvent) - def trigger_load_for_all_files(self, delay: int = None): # trigger event for every file diff --git a/HABApp/runtime/folder_watcher/simpleasyncfileevent.py b/HABApp/runtime/folder_watcher/simpleasyncfileevent.py new file mode 100644 index 00000000..abe93361 --- /dev/null +++ b/HABApp/runtime/folder_watcher/simpleasyncfileevent.py @@ -0,0 +1,77 @@ +import asyncio +import typing +from pathlib import Path +from threading import Lock + +from watchdog.events import FileSystemEvent, FileSystemEventHandler + +import HABApp +from HABApp.core.wrapper import ignore_exception + +LOCK = Lock() + + +class SimpleAsyncEventHandler(FileSystemEventHandler): + def __init__(self, target_func: typing.Callable[[Path], typing.Any], file_ending: str, worker_factory=None): + self.__target_func = target_func + + assert isinstance(file_ending, str), type(file_ending) + self.file_ending = file_ending + + # Possibility to use a wrapper to load files + # do not reuse an instantiated WrappedFunction because it will throw errors in the traceback module + self.__worker_factory = worker_factory + + # Pending events + self.__tasks: typing.Dict[str, asyncio.Future] = {} + + def __execute(self, dst: str): + if self.__worker_factory is None: + return self.__target_func(Path(dst)) + return self.__worker_factory(self.__target_func)(Path(dst)) + + def dispatch(self, event): + self.on_any_event(event) + + def on_any_event(self, event: FileSystemEvent): + # we don't process directory events + if event.is_directory: + return None + + src = event.src_path + if src.endswith(self.file_ending): + self.process_dst(src) + + # moved events have a dst, so we process it, too + if hasattr(event, 'dest_path'): + dst = event.dest_path + if dst.endswith(self.file_ending): + self.process_dst(dst) + return None + + @ignore_exception + def process_dst(self, dst: str): + # this has to be thread safe! + with LOCK: + try: + # cancel already running Task + self.__tasks[dst].cancel() + except KeyError: + pass + # and create a new one + self.__tasks[dst] = asyncio.run_coroutine_threadsafe(self.event_waiter(dst), loop=HABApp.core.const.loop) + + @ignore_exception + async def event_waiter(self, dst: str): + try: + # debounce time + await asyncio.sleep(0.4) + + # remove debounce task for target file + with LOCK: + _ = self.__tasks.pop(dst, None) + + # trigger file event + self.__execute(dst) + except asyncio.CancelledError: + pass diff --git a/HABApp/runtime/folder_watcher/simplefileevent.py b/HABApp/runtime/folder_watcher/simplefileevent.py deleted file mode 100644 index 0833fc0b..00000000 --- a/HABApp/runtime/folder_watcher/simplefileevent.py +++ /dev/null @@ -1,70 +0,0 @@ -from pathlib import Path - -from watchdog.events import FileSystemEventHandler, FileMovedEvent, FileCreatedEvent, \ - FileDeletedEvent, FileModifiedEvent - - -class FileEventTarget: - def on_file_added(self, path: Path): - raise NotImplementedError() - - def on_file_changed(self, path: Path): - raise NotImplementedError() - - def on_file_removed(self, path: Path): - raise NotImplementedError() - - -class SimpleFileEventHandler(FileSystemEventHandler): - def __init__(self, event_target, file_ending: str, worker_factory=None): - assert isinstance(event_target, FileEventTarget), type(event_target) - self.target = event_target - - assert isinstance(file_ending, str), type(file_ending) - self.__file_ending = file_ending - - # Possibility to use a wrapper to load files - # do not reuse an instantiated WrappedFunction because it will throw errors in the traceback module - self.__worker_factory = worker_factory - - def _get_func(self, func): - if self.__worker_factory is None: - return func - return self.__worker_factory(func) - - def on_deleted(self, event): - if not isinstance(event, FileDeletedEvent): - return None - - if not event.src_path.endswith(self.__file_ending): - return None - - self._get_func(self.target.on_file_removed)(Path(event.src_path)) - - def on_modified(self, event): - if not isinstance(event, FileModifiedEvent): - return None - - if not event.src_path.endswith(self.__file_ending): - return None - - self._get_func(self.target.on_file_changed)(Path(event.src_path)) - - def on_created(self, event): - if not isinstance(event, FileCreatedEvent): - return None - - if not event.src_path.endswith(self.__file_ending): - return None - - self._get_func(self.target.on_file_added)(Path(event.src_path)) - - def on_moved(self, event): - if not isinstance(event, FileMovedEvent): - return None - - if event.src_path.endswith(self.__file_ending): - self._get_func(self.target.on_file_removed)(Path(event.src_path)) - - if event.dest_path.endswith(self.__file_ending): - self._get_func(self.target.on_file_added)(Path(event.dest_path)) diff --git a/HABApp/runtime/runtime.py b/HABApp/runtime/runtime.py index 5fc4eb68..ac6d9168 100644 --- a/HABApp/runtime/runtime.py +++ b/HABApp/runtime/runtime.py @@ -60,7 +60,7 @@ def startup(self, config_folder: Path): self.folder_watcher.watch_folder( folder=config_folder, file_ending='.yml', - event_target=self.config_loader + target_func=self.config_loader.on_file_event ) # folder watcher rules @@ -73,13 +73,13 @@ def startup(self, config_folder: Path): if params_enabled: param_watcher = self.folder_watcher.watch_folder_habapp_events( folder=HABApp.CONFIG.directories.param, file_ending='.yml', - habapp_topic=HABApp.core.const.topics.PARAM, watch_subfolders=False + habapp_topic=HABApp.core.const.topics.PARAM, watch_subfolders=True ) # load all param files through the worker HABApp.core.WrappedFunction(param_watcher.trigger_load_for_all_files, name='Load all parameter files').run() - @HABApp.util.log_exception + @HABApp.core.wrapper.log_exception def get_async(self): return asyncio.gather( self.async_http.create_client(HABApp.core.const.loop), diff --git a/HABApp/util/__init__.py b/HABApp/util/__init__.py index a9205bc4..8c074b3c 100644 --- a/HABApp/util/__init__.py +++ b/HABApp/util/__init__.py @@ -1,5 +1,3 @@ -from .wrapper import log_exception, ignore_exception, ExceptionToHABApp - from .timeframe import TimeFrame from .counter_item import CounterItem diff --git a/HABApp/util/multimode_item.py b/HABApp/util/multimode_item.py index f884369b..29dd28ca 100644 --- a/HABApp/util/multimode_item.py +++ b/HABApp/util/multimode_item.py @@ -15,6 +15,10 @@ class MultiModeValue: a given timedelta on the next recalculation :ivar typing.Optional[str] auto_disable_on: Automatically disable this mode if the state with lower priority is ``>``, ``>=``, ``<``, ``<=``, ``==`` or ``!=`` than the own value + :vartype auto_disable_func: typing.Optional[typing.Callable[[typing.Any, typing.Any], bool]] + :ivar auto_disable_func: Function which can be used to disable this mode. Any function that accepts two + Arguments can be used. First arg is value with lower priority, second argument is own value. + Return ``True`` to disable this mode. :vartype calc_value_func: typing.Optional[typing.Callable[[typing.Any, typing.Any], typing.Any]] :ivar calc_value_func: Function to calculate the new value (e.g. ``min`` or ``max``). Any function that accepts two Arguments can be used. First arg is value with lower priority, second argument is own value. @@ -24,7 +28,8 @@ class MultiModeValue: '==': operator.eq, '!=': operator.ne, } - def __init__(self, parent, name: str, initial_value=None, auto_disable_on=None, auto_disable_after=None, + def __init__(self, parent, name: str, initial_value=None, + auto_disable_on=None, auto_disable_after=None, auto_disable_func=None, calc_value_func=None): assert isinstance(parent, MultiModeItem), type(parent) @@ -48,6 +53,7 @@ def __init__(self, parent, name: str, initial_value=None, auto_disable_on=None, assert auto_disable_on is None or auto_disable_on in MultiModeValue.DISABLE_OPERATORS, auto_disable_on self.auto_disable_after: typing.Optional[datetime.timedelta] = auto_disable_after self.auto_disable_on: typing.Optional[str] = auto_disable_on + self.auto_disable_func: typing.Optional[typing.Callable[[typing.Any, typing.Any], bool]] = auto_disable_func self.calc_value_func: typing.Optional[typing.Callable[[typing.Any, typing.Any], typing.Any]] = calc_value_func @@ -133,6 +139,15 @@ def calculate_value(self, value_with_lower_priority: typing.Any) -> typing.Any: self.__parent.log(logging.INFO, f'{self.__name} disabled ' f'({value_with_lower_priority}{self.auto_disable_on}{self.__value})!') + # provide user function which can disable a mode + if self.auto_disable_func is not None: + if self.auto_disable_func(value_with_lower_priority, self.__value) is True: + self.__enabled = False + self.last_update = datetime.datetime.now() + self.__parent.log(logging.INFO, f'{self.__name} disabled ' + f'({value_with_lower_priority}{self.auto_disable_on}{self.__value})!') + + # check if we may have disabled this mode if not self.__enabled: return value_with_lower_priority @@ -174,6 +189,7 @@ def create_mode( self, name: str, priority: int, initial_value: typing.Optional[typing.Any] = None, auto_disable_on: typing.Optional[str] = None, auto_disable_after: typing.Optional[datetime.timedelta] = None, + auto_disable_func: typing.Optional[typing.Callable[[typing.Any, typing.Any], bool]] = None, calc_value_func: typing.Optional[typing.Callable[[typing.Any, typing.Any], typing.Any]] = None ) -> MultiModeValue: """Create a new mode with priority @@ -185,6 +201,8 @@ def create_mode( See :attr:`~HABApp.util.multimode_item.MultiModeValue` :param auto_disable_after: Automatically disable the mode after a timedelta if a recalculate is done See :attr:`~HABApp.util.multimode_item.MultiModeValue` + :param auto_disable_func: Automatically disable the mode with a custom function + See :attr:`~HABApp.util.multimode_item.MultiModeValue` :param calc_value_func: See :attr:`~HABApp.util.multimode_item.MultiModeValue` :return: The newly created MultiModeValue """ @@ -197,6 +215,7 @@ def create_mode( self, name, initial_value=initial_value, auto_disable_on=auto_disable_on, auto_disable_after=auto_disable_after, + auto_disable_func=auto_disable_func, calc_value_func=calc_value_func ) self.__values_by_prio[priority] = ret @@ -216,7 +235,10 @@ def get_mode(self, name: str) -> MultiModeValue: :param name: name of the mode (case insensitive) :return: The requested MultiModeValue """ - return self.__values_by_name[name.lower()] + try: + return self.__values_by_name[name.lower()] + except KeyError: + raise KeyError(f'Unknown mode "{name}"! Available: {", ".join(self.__values_by_name.keys())}') from None def calculate_value(self) -> typing.Any: """Recalculate the output value and post the state to the event bus (if it is not None) diff --git a/_doc/util.rst b/_doc/util.rst index 3bda6bb5..035faf8c 100644 --- a/_doc/util.rst +++ b/_doc/util.rst @@ -158,6 +158,9 @@ Advanced Example print('-' * 80) item.get_mode('manual').auto_disable_on = '>=' # disable when low priority value >= mode value + # A custom function can also disable the mode: + # item.get_mode('manual').auto_disable_func = lambda low, own: low >= own + item.get_mode('Automatic').set_value(11) # <-- manual now gets disabled because item.get_mode('Automatic').set_value(4) # the lower priority value is >= itself diff --git a/setup.py b/setup.py index 107a90c9..5e46425b 100644 --- a/setup.py +++ b/setup.py @@ -67,6 +67,7 @@ def load_version() -> str: "Operating System :: OS Independent", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3 :: Only", "Topic :: Home Automation" ], diff --git a/tests/test_utils/test_multivalue.py b/tests/test_utils/test_multivalue.py index b7f41e3d..9f2bab47 100644 --- a/tests/test_utils/test_multivalue.py +++ b/tests/test_utils/test_multivalue.py @@ -1,5 +1,6 @@ -from HABApp.util import MultiModeItem +import pytest +from HABApp.util import MultiModeItem from ..test_core import ItemTests @@ -56,3 +57,32 @@ def test_auto_disable_on(): m1.set_value(59) assert p.value == 59 + + +def test_auto_disable_func(): + p = MultiModeItem('TestItem') + m1 = p.create_mode('modea', 1, 50) + m2 = p.create_mode('modeb', 2, 60, auto_disable_func=lambda low, s: low == 40) + + m2.set_value(60) + assert p.value == 60 + assert m2.enabled is True + + m1.set_value(40) + + assert p.value == 40 + assert m2.enabled is False + + m1.set_value(50) + assert p.value == 50 + assert m2.enabled is False + + +def test_unknown(): + p = MultiModeItem('asdf') + with pytest.raises(KeyError): + p.get_mode('asdf') + + p.create_mode('mode', 1, 50) + with pytest.raises(KeyError): + p.get_mode('asdf')