Reworked file handling

requirements_setup.txt

@@ -1,7 +1,7 @@
 aiohttp     == 3.11.10
 pydantic    == 2.10.3
 bidict      == 0.23.1
-watchdog    == 6.0.0
+watchfiles  == 1.0.3
 ujson       == 5.10.0
 aiomqtt     == 2.3.0
 

src/HABApp/__check_dependency_packages__.py

@@ -19,7 +19,7 @@ def get_dependencies() -> list[str]:
         'pydantic',
         'stack_data',
         'voluptuous',
-        'watchdog',
+        'watchfiles',
         'ujson',
         'immutables',
         'javaproperties',

src/HABApp/config/__init__.py

@@ -6,4 +6,4 @@
 
 # isort: split
 
-from .loader import load_config
+from .loader import setup_habapp_configuration

src/HABApp/config/loader.py

@@ -5,10 +5,11 @@
 import eascheduler
 import pydantic
 
-import HABApp
 from HABApp import __version__
 from HABApp.config.config import CONFIG
 from HABApp.config.logging import HABAppQueueHandler, load_logging_file
+from HABApp.core import shutdown
+from HABApp.core.internals.proxy.proxies import uses_file_manager
 
 from .debug import setup_debug
 from .errors import AbsolutePathExpected, InvalidConfigError
@@ -19,7 +20,10 @@
 log = logging.getLogger('HABApp.Config')
 
 
-def load_config(config_folder: Path) -> None:
+file_manager = uses_file_manager()
+
+
+def setup_habapp_configuration(config_folder: Path) -> None:
 
     CONFIG.set_file_path(config_folder / 'config.yml')
 
@@ -40,29 +44,27 @@ def load_config(config_folder: Path) -> None:
     if not loaded_logging:
         load_logging_cfg(logging_cfg_path)
 
+    shutdown.register(stop_queue_handlers, msg='Stop logging queue handlers', last=True)
+
     setup_debug()
 
-    # Watch folders, so we can reload the config on the fly
-    filter = HABApp.core.files.watcher.FileEndingFilter('.yml')
-    watcher = HABApp.core.files.watcher.AggregatingAsyncEventHandler(
-        config_folder, config_files_changed, filter, watch_subfolders=False
-    )
-    HABApp.core.files.watcher.add_folder_watch(watcher)
+    watcher = file_manager.get_file_watcher()
+    watcher.watch_file('config.log_file', config_file_changed, config_folder / 'logging.yml', habapp_internal=True)
+    watcher.watch_file('config.cfg_file', config_file_changed, config_folder / 'config.yml', habapp_internal=True)
 
-    HABApp.core.shutdown.register(stop_queue_handlers, last=True, msg='Stopping logging threads')
     CONFIG.habapp.logging.subscribe_for_changes(set_flush_delay)
 
 
 def set_flush_delay() -> None:
     HABAppQueueHandler.FLUSH_DELAY = CONFIG.habapp.logging.flush_every
 
 
-async def config_files_changed(paths: list[Path]) -> None:
-    for path in paths:
-        if path.name == 'config.yml':
-            load_habapp_cfg()
-        if path.name == 'logging.yml':
-            load_logging_cfg(path)
+async def config_file_changed(path: str) -> None:
+    file = Path(path)
+    if file.name == 'config.yml':
+        load_habapp_cfg()
+    if file.name == 'logging.yml':
+        load_logging_cfg(file)
 
 
 def load_habapp_cfg(do_print=False) -> None:

src/HABApp/core/events/habapp_events.py

@@ -8,6 +8,9 @@ def __init__(self, name: str) -> None:
     def __repr__(self) -> str:
         return f'<{self.__class__.__name__} filename: {self.name}>'
 
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, self.__class__) and self.name == other.name
+
 
 class RequestFileLoadEvent(__FileEventBase):
     """Request (re-) loading of the specified file

src/HABApp/core/files/__init__.py

@@ -1,8 +1,2 @@
-from . import errors
-
-from . import watcher
-from . import file
-from . import folders
-from . import manager
-
-from .setup import setup
+from .watcher import HABAppFileWatcher, FolderDispatcher, FileDispatcher
+from .manager import FileManager

src/HABApp/core/files/errors.py

@@ -1,8 +1,5 @@
-from collections.abc import Iterable as _Iterable
-
-
 class CircularReferenceError(Exception):
-    def __init__(self, stack: _Iterable[str]) -> None:
+    def __init__(self, stack: tuple[str, ...]) -> None:
         self.stack = stack
 
     def __repr__(self) -> str:

src/HABApp/core/files/file.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+from enum import Enum, auto
+from hashlib import blake2b
+from typing import TYPE_CHECKING, Final
+
+from HABApp.core.files.errors import AlreadyHandledFileError, CircularReferenceError, DependencyDoesNotExistError
+from HABApp.core.files.file_properties import FileProperties
+from HABApp.core.wrapper import process_exception
+
+
+if TYPE_CHECKING:
+    import logging
+    from pathlib import Path
+
+    from HABApp.core.files.manager import FileManager, FileTypeHandler
+
+
+class FileState(Enum):
+    LOADED = auto()
+    FAILED = auto()
+
+    DEPENDENCIES_OK = auto()
+    DEPENDENCIES_MISSING = auto()
+    DEPENDENCIES_ERROR = auto()
+
+    PROPERTIES_INVALID = auto()  # Properties could not be parsed
+
+    # initial and last state
+    PENDING = auto()
+    REMOVED = auto()
+
+    def __str__(self) -> str:
+        return str(self.name)
+
+
+class HABAppFile:
+
+    @staticmethod
+    def create_checksum(text: str) -> bytes:
+        b = blake2b()
+        b.update(text.encode())
+        return b.digest()
+
+    def __init__(self, name: str, path: Path, checksum: bytes, properties: FileProperties | None) -> None:
+        self.name: Final = name
+        self.path: Final = path
+        self.checksum: Final = checksum
+        self.properties: Final = properties if properties is not None else FileProperties()
+        self._state: FileState = FileState.PENDING if properties is not None else FileState.PROPERTIES_INVALID
+
+    def __repr__(self) -> str:
+        return f'<{self.__class__.__name__} {self.name} state: {self._state}>'
+
+    def set_state(self, new_state: FileState, manager: FileManager) -> None:
+        if self._state is new_state:
+            return None
+
+        self._state = new_state
+        manager.file_state_changed(self, str(new_state))
+
+    def _check_circ_refs(self, stack: tuple[str, ...], prop: str, manager: FileManager) -> None:
+        c: list[str] = getattr(self.properties, prop)
+        for f in c:
+            _stack = stack + (f, )
+            if f in stack:
+                raise CircularReferenceError(_stack)
+
+            next_file = manager.get_file(f)
+            if next_file is not None:
+                next_file._check_circ_refs(_stack, prop, manager)
+
+    def _check_properties(self, manager: FileManager, log: logging.Logger) -> None:
+        # check dependencies
+        missing = {name for name in self.properties.depends_on if manager.get_file(name) is None}
+        if missing:
+            one = len(missing) == 1
+            msg = (f'File {self.path} depends on file{"" if one else "s"} that '
+                   f'do{"es" if one else ""}n\'t exist: {", ".join(sorted(missing))}')
+            raise DependencyDoesNotExistError(msg)
+
+        # check reload
+        missing = {name for name in self.properties.reloads_on if manager.get_file(name) is None}
+        if missing:
+            one = len(missing) == 1
+            log.warning(f'File {self.path} reloads on file{"" if one else "s"} that '
+                        f'do{"es" if one else ""}n\'t exist: {", ".join(sorted(missing))}')
+
+    def check_properties(self, manager: FileManager, log: logging.Logger, *, log_msg: bool = False) -> None:
+        if self._state is not FileState.PENDING and self._state is not FileState.DEPENDENCIES_ERROR:
+            return None
+
+        try:
+            self._check_properties(manager, log)
+        except DependencyDoesNotExistError as e:
+            if log_msg:
+                log.error(e.msg)
+            return self.set_state(FileState.DEPENDENCIES_ERROR, manager)
+
+        try:
+            # check for circular references
+            self._check_circ_refs((self.name, ), 'depends_on', manager)
+            self._check_circ_refs((self.name, ), 'reloads_on', manager)
+        except CircularReferenceError as e:
+            log.error(f'Circular reference: {" -> ".join(e.stack)}')
+            return self.set_state(FileState.DEPENDENCIES_ERROR, manager)
+
+        # Check if we can already load it
+        new_state = FileState.DEPENDENCIES_OK if not self.properties.depends_on else FileState.DEPENDENCIES_MISSING
+        self.set_state(new_state, manager)
+        return None
+
+    def check_dependencies(self, manager: FileManager) -> None:
+        if self._state is not FileState.DEPENDENCIES_MISSING:
+            return None
+
+        for name in self.properties.depends_on:
+            if (file := manager.get_file(name)) is None:
+                return None
+            if file._state is not FileState.LOADED:
+                return None
+
+        self.set_state(FileState.DEPENDENCIES_OK, manager)
+        return None
+
+    def can_be_loaded(self) -> bool:
+        return self._state is FileState.DEPENDENCIES_OK
+
+    def can_be_removed(self) -> bool:
+        return self._state is FileState.REMOVED
+
+    async def load(self, handler: FileTypeHandler, manager: FileManager) -> None:
+        if not self.can_be_loaded():
+            msg = f'File {self.name} can not be loaded because current state is {self._state}!'
+            raise ValueError(msg)
+
+        try:
+            await handler.on_load(self.name, self.path)
+        except Exception as e:
+            if not isinstance(e, AlreadyHandledFileError):
+                process_exception(handler.on_load, e, logger=handler.logger)
+            self.set_state(FileState.FAILED, manager)
+            return None
+
+        self.set_state(FileState.LOADED, manager)
+        return None
+
+    async def unload(self, handler: FileTypeHandler, manager: FileManager) -> None:
+        try:
+            await handler.on_unload(self.name, self.path)
+        except Exception as e:
+            if not isinstance(e, AlreadyHandledFileError):
+                process_exception(handler.on_unload, e, logger=handler.logger)
+            self.set_state(FileState.FAILED, manager)
+            return None
+
+        self.set_state(FileState.REMOVED, manager)
+        return None
+
+    def file_state_changed(self, file: HABAppFile, manager: FileManager) -> None:
+        name = file.name
+        if name in self.properties.reloads_on:
+            self.set_state(FileState.PENDING, manager)