From 85ea60f009cc718753adb04d0cb4967552f8e84e Mon Sep 17 00:00:00 2001 From: Bob Steers Date: Thu, 7 Dec 2023 02:24:26 +0100 Subject: [PATCH] refactor everything to fastapi --- README.md | 2 +- brewblox_tilt/app_factory.py | 4 +- brewblox_tilt/broadcaster.py | 10 ++- brewblox_tilt/parser.py | 105 ++++-------------------- brewblox_tilt/scanner.py | 7 +- brewblox_tilt/stored.py | 150 +++++++++++++++++++++++++++-------- docker-compose.yml | 11 ++- entrypoint.sh | 5 +- parse_appenv.py | 10 ++- poetry.lock | 98 +++++++++++++---------- pyproject.toml | 10 +-- tasks.py | 15 +++- test/conftest.py | 87 +++++++++++++++++++- test/test_parser.py | 117 ++++++--------------------- test/test_stored.py | 75 +++++++++++------- 15 files changed, 404 insertions(+), 302 deletions(-) diff --git a/README.md b/README.md index 08c476d..250596e 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ sudo apt update && sudo apt install -y libbluetooth-dev To build a local Docker image: ```bash -poetry run invoke local-docker +poetry run invoke image ``` This builds the Python package and then the Dockerfile as `ghcr.io/brewblox/brewblox-tilt:local`. diff --git a/brewblox_tilt/app_factory.py b/brewblox_tilt/app_factory.py index d0481ff..8097881 100644 --- a/brewblox_tilt/app_factory.py +++ b/brewblox_tilt/app_factory.py @@ -4,7 +4,7 @@ from fastapi import FastAPI -from . import broadcaster, mqtt, parser, scanner, utils +from . import broadcaster, mqtt, parser, scanner, stored, utils LOGGER = logging.getLogger(__name__) @@ -23,6 +23,7 @@ def setup_logging(debug: bool): logging.getLogger('httpcore').setLevel(logging.WARN) logging.getLogger('uvicorn.access').setLevel(unimportant_level) logging.getLogger('uvicorn.error').disabled = True + logging.getLogger('bleak.backends.bluezdbus.manager').setLevel(unimportant_level) @asynccontextmanager @@ -42,6 +43,7 @@ def create_app() -> FastAPI: # Call setup functions for modules mqtt.setup() + stored.setup() parser.setup() scanner.setup() diff --git a/brewblox_tilt/broadcaster.py b/brewblox_tilt/broadcaster.py index a0921d7..e52208e 100644 --- a/brewblox_tilt/broadcaster.py +++ b/brewblox_tilt/broadcaster.py @@ -7,16 +7,22 @@ LOGGER = logging.getLogger(__name__) -class Broadcaster(): +class Broadcaster: def __init__(self): config = utils.get_config() self.name = config.name + self.scan_duration = max(config.scan_duration, 1) self.inactive_scan_interval = max(config.inactive_scan_interval, 0) self.active_scan_interval = max(config.active_scan_interval, 0) + self.state_topic = f'brewcast/state/{self.name}' self.history_topic = f'brewcast/history/{self.name}' + # Changes based on scan response + self.scan_interval = 1 + self.prev_num_messages = 0 + async def _run(self): mqtt_client = mqtt.CV.get() messages = await scanner.CV.get().scan(self.scan_duration) @@ -43,7 +49,7 @@ async def _run(self): if not messages: return - LOGGER.debug(messages) + LOGGER.debug('\n - '.join([str(v) for v in ['Messages:', *messages]])) # Publish history # Devices can share an event diff --git a/brewblox_tilt/parser.py b/brewblox_tilt/parser.py index 5811e50..011470c 100644 --- a/brewblox_tilt/parser.py +++ b/brewblox_tilt/parser.py @@ -1,13 +1,9 @@ -import csv -import json import logging from contextvars import ContextVar -from pathlib import Path -import numpy as np from pint import UnitRegistry -from . import const, mqtt, stored, utils +from . import const, stored, utils from .models import TiltEvent, TiltMessage, TiltTemperatureSync _UREG: ContextVar['UnitRegistry'] = ContextVar('parser.UnitRegistry') @@ -35,77 +31,13 @@ def sg_to_plato(sg: float | None) -> float | None: return round(plato, 3) -class Calibrator(): - def __init__(self, file: Path | str) -> None: - self.cal_polys: dict[str, np.poly1d] = {} - self.keys: set[str] = set() - self.path = Path(file) - self.path.parent.mkdir(parents=True, exist_ok=True) - self.path.touch() - self.path.chmod(0o666) - - cal_tables = {} - - # Load calibration CSV - with open(self.path, newline='') as f: - reader = csv.reader(f, delimiter=',') - for line in reader: - key = None # MAC or name - uncal = None - cal = None - - key = line[0].strip().lower() - - try: - uncal = float(line[1].strip()) - except ValueError: - LOGGER.warning(f'Uncalibrated value `{line[1]}` not a float. Ignoring line.') - continue - - try: - cal = float(line[2].strip()) - except ValueError: - LOGGER.warning(f'Calibrated value `{line[2]}` not a float. Ignoring line.') - continue - - self.keys.add(key) - data = cal_tables.setdefault(key, { - 'uncal': [], - 'cal': [], - }) - data['uncal'].append(uncal) - data['cal'].append(cal) - - # Use polyfit to fit a cubic polynomial curve to calibration values - # Then create a polynomical from the values produced by polyfit - for key, data in cal_tables.items(): - x = np.array(data['uncal']) - y = np.array(data['cal']) - z = np.polyfit(x, y, 3) - self.cal_polys[key] = np.poly1d(z) - - LOGGER.info(f'Calibration values loaded from `{self.path}`: keys={*self.cal_polys.keys(),}') - - def calibrated_value(self, key_candidates: list[str], value: float, ndigits=0) -> float | None: - # Use polynomials calculated above to calibrate values - # Both MAC and device name are valid keys in calibration files - # Check whether any of the given keys is present - for key in [k.lower() for k in key_candidates]: - if key in self.cal_polys: - return round(self.cal_polys[key](value), ndigits) - return None - - class EventDataParser(): def __init__(self): config = utils.get_config() self.lower_bound = config.lower_bound self.upper_bound = config.upper_bound - self.devconfig = stored.DeviceConfig(const.DEVICES_FILE_PATH) self.session_macs: set[str] = set() - self.sg_cal = Calibrator(const.SG_CAL_FILE_PATH) - self.temp_cal = Calibrator(const.TEMP_CAL_FILE_PATH) def _decode_event_data(self, event: TiltEvent) -> dict | None: """ @@ -152,13 +84,17 @@ def _parse_event(self, event: TiltEvent) -> TiltMessage | None: If the event is invalid, `message` is returned unchanged. """ + devices = stored.DEVICES.get() + sg_cal = stored.SG_CAL.get() + temp_cal = stored.TEMP_CAL.get() + decoded = self._decode_event_data(event) if decoded is None: return None color = decoded['color'] mac = event.mac.strip().replace(':', '').upper() - name = self.devconfig.lookup(mac, color) + name = devices.lookup(mac, color) if mac not in self.session_macs: self.session_macs.add(mac) @@ -171,15 +107,15 @@ def _parse_event(self, event: TiltEvent) -> TiltMessage | None: temp_digits = 1 if is_pro else 0 sg_digits = 4 if is_pro else 3 - cal_temp_f = self.temp_cal.calibrated_value([mac, name], - raw_temp_f, - temp_digits) + cal_temp_f = temp_cal.calibrated_value([mac, name], + raw_temp_f, + temp_digits) cal_temp_c = deg_f_to_c(cal_temp_f) raw_sg = decoded['sg'] - cal_sg = self.sg_cal.calibrated_value([mac, name], - raw_sg, - sg_digits) + cal_sg = sg_cal.calibrated_value([mac, name], + raw_sg, + sg_digits) raw_plato = sg_to_plato(raw_sg) cal_plato = sg_to_plato(cal_sg) @@ -209,7 +145,7 @@ def _parse_event(self, event: TiltEvent) -> TiltMessage | None: sync: list[TiltTemperatureSync] = [] - for src in self.devconfig.sync: + for src in devices.sync: sync_tilt = src.get('tilt') sync_type = src.get('type') sync_service = src.get('service') @@ -238,22 +174,11 @@ def parse(self, events: list[TiltEvent]) -> list[TiltMessage]: Converts a list of Tilt events into a list of Tilt message. Invalid events are excluded. """ - messages = [self._parse_event(evt) for evt in events] - self.devconfig.commit() + with stored.DEVICES.get().autocommit(): + messages = [self._parse_event(evt) for evt in events] return [msg for msg in messages if msg is not None] - def apply_custom_names(self, names: dict[str, str]) -> None: - self.devconfig.apply_custom_names(names) - self.devconfig.commit() - def setup(): - config = utils.get_config() - mqtt_client = mqtt.CV.get() - _UREG.set(UnitRegistry()) CV.set(EventDataParser()) - - @mqtt_client.subscribe(f'brewcast/tilt/{config.name}/names') - async def on_names_change(client, topic, payload, qos, properties): - CV.get().apply_custom_names(json.loads(payload)) diff --git a/brewblox_tilt/scanner.py b/brewblox_tilt/scanner.py index ad5dc5d..610ae4e 100644 --- a/brewblox_tilt/scanner.py +++ b/brewblox_tilt/scanner.py @@ -86,6 +86,7 @@ def __init__(self, simulated: str) -> None: if color.upper() == simulated.upper() ), '') self.mac = self.uuid.replace('-', '').upper()[:12] + LOGGER.info(f'Simulation: {simulated}={self.mac}') self.interval = 1 self.temp_f = 68 @@ -99,10 +100,10 @@ def update(self) -> TiltEvent: return TiltEvent(mac=self.mac, uuid=self.uuid, - major=self.temp_f, - minor=self.raw_sg, + major=int(self.temp_f), + minor=int(self.raw_sg), txpower=0, - rssi=self.rssi) + rssi=int(self.rssi)) class SimulatedScanner(BaseScanner): diff --git a/brewblox_tilt/stored.py b/brewblox_tilt/stored.py index 1281abd..2d438cd 100644 --- a/brewblox_tilt/stored.py +++ b/brewblox_tilt/stored.py @@ -1,22 +1,29 @@ +import csv +import json import logging import re +from contextlib import contextmanager +from contextvars import ContextVar from pathlib import Path -from typing import Union +import numpy as np from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap, CommentedSeq -from . import const +from . import const, mqtt, utils LOGGER = logging.getLogger(__name__) +DEVICES: ContextVar['DeviceConfig'] = ContextVar('stored.DeviceConfig') +SG_CAL: ContextVar['Calibrator'] = ContextVar('stored.Calibrator.sg') +TEMP_CAL: ContextVar['Calibrator'] = ContextVar('stored.Calibrator.temp') -class DeviceConfig(): - def __init__(self, file: Union[Path, str]) -> None: + +class DeviceConfig: + def __init__(self, file: Path) -> None: self.path = Path(file) self.yaml = YAML() self.changed = False - self.device_config = {} self.path.parent.mkdir(parents=True, exist_ok=True) self.path.touch() @@ -26,31 +33,24 @@ def __init__(self, file: Union[Path, str]) -> None: self.device_config.setdefault('names', CommentedMap()) self.device_config.setdefault('sync', CommentedSeq()) - if not self.sync: - self.sync.append({ - 'type': 'TempSensorExternal', - 'tilt': 'ExampleTilt', - 'service': 'example-spark-service', - 'block': 'Example Block Name' - }) - self.changed = True - - for mac, name in list(self.names.items()): - if not re.match(const.DEVICE_NAME_PATTERN, name): - sanitized = re.sub(const.INVALID_NAME_CHAR_PATTERN, '_', name) or 'Unknown' - LOGGER.warning(f'Sanitizing invalid device name: {mac=} {name=}, {sanitized=}.') - self.names[mac] = sanitized + with self.autocommit(): + if not self.sync: + self.sync.append({ + 'type': 'TempSensorExternal', + 'tilt': 'ExampleTilt', + 'service': 'example-spark-service', + 'block': 'Example Block Name' + }) self.changed = True - LOGGER.info(f'Device config loaded from `{self.path}`: {str(dict(self.names))}') - - @property - def names(self) -> dict[str, str]: - return self.device_config['names'] + for mac, name in list(self.names.items()): + if not re.match(const.DEVICE_NAME_PATTERN, name): + sanitized = re.sub(const.INVALID_NAME_CHAR_PATTERN, '_', name) or 'Unknown' + LOGGER.warning(f'Sanitizing invalid device name: {mac=} {name=}, {sanitized=}.') + self.names[mac] = sanitized + self.changed = True - @property - def sync(self) -> list[dict[str, str]]: - return self.device_config['sync'] + LOGGER.info(f'Device config loaded from `{self.path}`: {str(dict(self.names))}') def _assign(self, base_name: str) -> str: used: set[str] = set(self.names.values()) @@ -68,6 +68,23 @@ def _assign(self, base_name: str) -> str: # If we have >1000 entries for a given base name, something went wrong raise RuntimeError('Name increment attempts exhausted') # pragma: no cover + @property + def names(self) -> dict[str, str]: + return self.device_config['names'] + + @property + def sync(self) -> list[dict[str, str]]: + return self.device_config['sync'] + + @contextmanager + def autocommit(self): + try: + yield + finally: + if self.changed: + self.yaml.dump(self.device_config, self.path) + self.changed = False + def lookup(self, mac: str, base_name: str) -> str: if not re.match(const.NORMALIZED_MAC_PATTERN, mac): raise ValueError(f'{mac} is not a normalized device MAC address.') @@ -94,7 +111,78 @@ def apply_custom_names(self, names: dict[str, str]): self.names[mac] = name self.changed = True - def commit(self): - if self.changed: - self.yaml.dump(self.device_config, self.path) - self.changed = False + +class Calibrator: + def __init__(self, file: Path | str) -> None: + self.cal_polys: dict[str, np.poly1d] = {} + self.keys: set[str] = set() + self.path = Path(file) + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.touch() + self.path.chmod(0o666) + + cal_tables = {} + + # Load calibration CSV + with open(self.path, newline='') as f: + reader = csv.reader(f, delimiter=',') + for line in reader: + key = None # MAC or name + uncal = None + cal = None + + key = line[0].strip().lower() + + try: + uncal = float(line[1].strip()) + except ValueError: + LOGGER.warning(f'Uncalibrated value `{line[1]}` not a float. Ignoring line.') + continue + + try: + cal = float(line[2].strip()) + except ValueError: + LOGGER.warning(f'Calibrated value `{line[2]}` not a float. Ignoring line.') + continue + + self.keys.add(key) + data = cal_tables.setdefault(key, { + 'uncal': [], + 'cal': [], + }) + data['uncal'].append(uncal) + data['cal'].append(cal) + + # Use polyfit to fit a cubic polynomial curve to calibration values + # Then create a polynomical from the values produced by polyfit + for key, data in cal_tables.items(): + x = np.array(data['uncal']) + y = np.array(data['cal']) + z = np.polyfit(x, y, 3) + self.cal_polys[key] = np.poly1d(z) + + LOGGER.info(f'Calibration values loaded from `{self.path}`: keys={*self.cal_polys.keys(),}') + + def calibrated_value(self, key_candidates: list[str], value: float, ndigits=0) -> float | None: + # Use polynomials calculated above to calibrate values + # Both MAC and device name are valid keys in calibration files + # Check whether any of the given keys is present + for key in [k.lower() for k in key_candidates]: + if key in self.cal_polys: + return round(self.cal_polys[key](value), ndigits) + return None + + +def setup(): + config = utils.get_config() + mqtt_client = mqtt.CV.get() + + DEVICES.set(DeviceConfig(const.DEVICES_FILE_PATH)) + SG_CAL.set(Calibrator(const.SG_CAL_FILE_PATH)) + TEMP_CAL.set(Calibrator(const.TEMP_CAL_FILE_PATH)) + + @mqtt_client.subscribe(f'brewcast/tilt/{config.name}/names') + async def on_names_change(client, topic, payload, qos, properties): + devconfig = DEVICES.get() + with devconfig.autocommit(): + devconfig.apply_custom_names(json.loads(payload)) diff --git a/docker-compose.yml b/docker-compose.yml index 988ea4f..e67f668 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,12 +8,21 @@ services: # Manually build the "local" image before use image: ghcr.io/brewblox/brewblox-tilt:local privileged: true - command: --debug + environment: + - BREWBLOX_DEBUG=True + - UVICORN_RELOAD=True + # - BREWBLOX_SIMULATE=["Mauve", "Lila"] restart: unless-stopped volumes: - type: bind source: ./brewblox_tilt target: /app/brewblox_tilt + - type: bind + source: ./entrypoint.sh + target: /app/entrypoint.sh + - type: bind + source: ./parse_appenv.py + target: /app/parse_appenv.py - type: bind source: ./share target: /share diff --git a/entrypoint.sh b/entrypoint.sh index 45946f6..3f5039f 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,7 +3,10 @@ set -euo pipefail python3 ./parse_appenv.py "$@" >.appenv +# This service does not have a REST API +# We use the scaffolding for convenience, +# but don't bind to a port exec uvicorn \ - --uds /run/dummy.sock \ + --uds /run/tilt_dummy.sock \ --factory \ brewblox_tilt.app_factory:create_app diff --git a/parse_appenv.py b/parse_appenv.py index 45cba7a..e7885d2 100644 --- a/parse_appenv.py +++ b/parse_appenv.py @@ -1,4 +1,5 @@ import argparse +import json import shlex import sys @@ -31,5 +32,12 @@ def parse_cmd_args(raw_args: list[str]) -> tuple[argparse.Namespace, list[str]]: print(f'WARNING: ignoring unknown CMD arguments: {unknown}', file=sys.stderr) output = [f'brewblox_{k}={shlex.quote(str(v))}' for k, v in vars(args).items() - if v is not None and v is not False] + if v is not None + and v is not False + and k != 'simulate'] print(*output, sep='\n') + + # Special exception for list variables + if args.simulate: + sim_names = json.dumps(list(args.simulate)) + print(f"brewblox_simulate='{sim_names}'") diff --git a/poetry.lock b/poetry.lock index 0f70b8f..3605536 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,20 +31,6 @@ doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd- test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] trio = ["trio (<0.22)"] -[[package]] -name = "asgi-lifespan" -version = "2.1.0" -description = "Programmatic startup/shutdown of ASGI apps." -optional = false -python-versions = ">=3.7" -files = [ - {file = "asgi-lifespan-2.1.0.tar.gz", hash = "sha256:5e2effaf0bfe39829cf2d64e7ecc47c7d86d676a6599f7afba378c31f5e3a308"}, - {file = "asgi_lifespan-2.1.0-py3-none-any.whl", hash = "sha256:ed840706680e28428c01e14afb3875d7d76d3206f3d5b2f2294e059b5c23804f"}, -] - -[package.dependencies] -sniffio = "*" - [[package]] name = "attrs" version = "23.1.0" @@ -116,6 +102,17 @@ files = [ {file = "bleak_winrt-1.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:63130c11acfe75c504a79c01f9919e87f009f5e742bfc7b7a5c2a9c72bf591a7"}, ] +[[package]] +name = "certifi" +version = "2023.11.17" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, + {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, +] + [[package]] name = "click" version = "8.1.7" @@ -368,6 +365,27 @@ files = [ {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, ] +[[package]] +name = "httpcore" +version = "1.0.2" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.2-py3-none-any.whl", hash = "sha256:096cc05bca73b8e459a1fc3dcf585148f63e534eae4339559c9b8a8d6399acc7"}, + {file = "httpcore-1.0.2.tar.gz", hash = "sha256:9fc092e4799b26174648e54b74ed5f683132a464e95643b226e00c2ed2fa6535"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<0.23.0)"] + [[package]] name = "httptools" version = "0.6.1" @@ -416,6 +434,30 @@ files = [ [package.extras] test = ["Cython (>=0.29.24,<0.30.0)"] +[[package]] +name = "httpx" +version = "0.25.2" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx-0.25.2-py3-none-any.whl", hash = "sha256:a05d3d052d9b2dfce0e3896636467f8a5342fb2b902c819428e1ac65413ca118"}, + {file = "httpx-0.25.2.tar.gz", hash = "sha256:8b8fcaa0c8ea7b05edd69a094e63a2094c4efcb48129fb757361bc423c0ad9e8"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] + [[package]] name = "idna" version = "3.6" @@ -460,32 +502,6 @@ files = [ {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] -[[package]] -name = "ninja" -version = "1.10.2.3" -description = "Ninja is a small build system with a focus on speed" -optional = false -python-versions = "*" -files = [ - {file = "ninja-1.10.2.3-py2.py3-none-macosx_10_9_universal2.macosx_10_9_x86_64.macosx_11_0_arm64.macosx_11_0_universal2.whl", hash = "sha256:d5e0275d28997a750a4f445c00bdd357b35cc334c13cdff13edf30e544704fbd"}, - {file = "ninja-1.10.2.3-py2.py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ea785bf6a15727040835256577239fa3cf5da0d60e618c307aa5efc31a1f0ce"}, - {file = "ninja-1.10.2.3-py2.py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29570a18d697fc84d361e7e6330f0021f34603ae0fcb0ef67ae781e9814aae8d"}, - {file = "ninja-1.10.2.3-py2.py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:21a1d84d4c7df5881bfd86c25cce4cf7af44ba2b8b255c57bc1c434ec30a2dfc"}, - {file = "ninja-1.10.2.3-py2.py3-none-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9ca8dbece144366d5f575ffc657af03eb11c58251268405bc8519d11cf42f113"}, - {file = "ninja-1.10.2.3-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:279836285975e3519392c93c26e75755e8a8a7fafec9f4ecbb0293119ee0f9c6"}, - {file = "ninja-1.10.2.3-py2.py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:cc8b31b5509a2129e4d12a35fc21238c157038022560aaf22e49ef0a77039086"}, - {file = "ninja-1.10.2.3-py2.py3-none-musllinux_1_1_i686.whl", hash = "sha256:688167841b088b6802e006f911d911ffa925e078c73e8ef2f88286107d3204f8"}, - {file = "ninja-1.10.2.3-py2.py3-none-musllinux_1_1_ppc64le.whl", hash = "sha256:840a0b042d43a8552c4004966e18271ec726e5996578f28345d9ce78e225b67e"}, - {file = "ninja-1.10.2.3-py2.py3-none-musllinux_1_1_s390x.whl", hash = "sha256:84be6f9ec49f635dc40d4b871319a49fa49b8d55f1d9eae7cd50d8e57ddf7a85"}, - {file = "ninja-1.10.2.3-py2.py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:6bd76a025f26b9ae507cf8b2b01bb25bb0031df54ed685d85fc559c411c86cf4"}, - {file = "ninja-1.10.2.3-py2.py3-none-win32.whl", hash = "sha256:740d61fefb4ca13573704ee8fe89b973d40b8dc2a51aaa4e9e68367233743bb6"}, - {file = "ninja-1.10.2.3-py2.py3-none-win_amd64.whl", hash = "sha256:0560eea57199e41e86ac2c1af0108b63ae77c3ca4d05a9425a750e908135935a"}, - {file = "ninja-1.10.2.3.tar.gz", hash = "sha256:e1b86ad50d4e681a7dbdff05fc23bb52cb773edb90bc428efba33fa027738408"}, -] - -[package.extras] -test = ["codecov (>=2.0.5)", "coverage (>=4.2)", "flake8 (>=3.0.4)", "pytest (>=4.5.0)", "pytest-cov (>=2.7.1)", "pytest-runner (>=5.1)", "pytest-virtualenv (>=1.7.0)", "virtualenv (>=15.0.3)"] - [[package]] name = "numpy" version = "1.25.2" @@ -1325,4 +1341,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "~3.11" -content-hash = "4adc47ce513b1e7e2cc598bede161e41b5b2a372ee99d41364645ba2f308482e" +content-hash = "edbd42b6da865641a0f76e3914610e3432ae82ae1abef9551d2011c9a049678f" diff --git a/pyproject.toml b/pyproject.toml index ee87305..ff794f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,11 +16,10 @@ Pint = "^0.22" numpy = "1.25.2" bleak = "^0.21.1" construct = "^2.10.68" -ninja = "1.10.2.3" pydantic-settings = "^2.1.0" fastapi = "^0.104.1" fastapi-mqtt = "^2.0.0" -uvicorn = {extras = ["standard"], version = "^0.24.0.post1"} +uvicorn = { extras = ["standard"], version = "^0.24.0.post1" } [tool.poetry.group.dev.dependencies] pytest-cov = "*" @@ -33,8 +32,8 @@ pytest = "*" invoke = "*" flake8-pyproject = "*" pytest-asyncio = "*" -asgi-lifespan = "*" pytest-docker = "*" +httpx = "^0.25.2" [build-system] requires = ["poetry-core>=1.0.0"] @@ -44,11 +43,10 @@ build-backend = "poetry.core.masonry.api" asyncio_mode = "auto" addopts = """ --ignore=app/ - --ignore=victoria/ - --cov=brewblox_history + --ignore=share/ + --cov=brewblox_tilt --cov-branch --cov-report=term-missing:skip-covered - --cov-fail-under=100 --no-cov-on-fail --durations=3 """ diff --git a/tasks.py b/tasks.py index b725f7d..213f11e 100644 --- a/tasks.py +++ b/tasks.py @@ -5,6 +5,19 @@ ROOT = Path(__file__).parent.resolve() +@task +def testclean(ctx: Context): + """ + Cleans up leftover test containers. + Container cleanup is normally done in test fixtures. + This is skipped if debugged tests are stopped halfway. + """ + result = ctx.run('docker ps -aq --filter "name=pytest"', hide='stdout') + containers = result.stdout.strip().replace('\n', ' ') + if containers: + ctx.run(f'docker rm -f {containers}') + + @task def build(ctx: Context): with ctx.cd(ROOT): @@ -14,6 +27,6 @@ def build(ctx: Context): @task(pre=[build]) -def local_docker(ctx: Context, tag='local'): +def image(ctx: Context, tag='local'): with ctx.cd(ROOT): ctx.run(f'docker build -t ghcr.io/brewblox/brewblox-tilt:{tag} .') diff --git a/test/conftest.py b/test/conftest.py index 389144b..bdc0626 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -4,8 +4,11 @@ """ +import json import logging +from io import FileIO from pathlib import Path +from tempfile import NamedTemporaryFile, TemporaryDirectory from typing import Generator import pytest @@ -14,7 +17,7 @@ from pytest_docker.plugin import Services as DockerServices from starlette.testclient import TestClient -from brewblox_tilt import app_factory, utils +from brewblox_tilt import app_factory, const, utils from brewblox_tilt.models import ServiceConfig LOGGER = logging.getLogger(__name__) @@ -80,3 +83,85 @@ def app() -> FastAPI: def client(app: FastAPI) -> Generator[TestClient, None, None]: with TestClient(app=app, base_url='http://test') as c: yield c + + +@pytest.fixture +def tilt_macs() -> dict: + return { + 'red': 'AA7F97FC141E', + 'black': 'DD7F97FC141E', + 'purple': 'BB7F97FC141E', + } + + +@pytest.fixture +def config_dir(monkeypatch: pytest.MonkeyPatch) -> TemporaryDirectory: + d = TemporaryDirectory() + monkeypatch.setattr(const, 'CONFIG_DIR', Path(d.name)) + yield d + + +@pytest.fixture +def devices_file(monkeypatch: pytest.MonkeyPatch, tilt_macs: dict) -> FileIO: + f = NamedTemporaryFile() + f.write(json.dumps({ + 'names': { + tilt_macs['red']: 'Red', + tilt_macs['black']: 'Black', + tilt_macs['purple']: 'Ferment 1 Tilt', + } + }).encode()) + f.flush() + monkeypatch.setattr(const, 'DEVICES_FILE_PATH', Path(f.name)) + yield f + + +@pytest.fixture +def sgcal_file(monkeypatch: pytest.MonkeyPatch) -> FileIO: + f = NamedTemporaryFile() + f.writelines([ + f'{s}\n'.encode() + for s in [ + 'Black, 1.000, 2.001', + 'Black, 1.001, 2.002', + 'Black, 1.002, 2.003', + 'BLACK, 1.003, 2.004', + 'Black, 1, Many', + 'Black, Few, 2.005', + '' + '"Ferment 1 red", 1.000, 3.010', + '"Ferment 1 red", 1.001, 3.011', + '"Ferment 1 red", 1.002, 3.012', + '"Ferment 1 red", 1.003, 3.013', + '"Ferment 1 red", 1.004, 3.014', + ]]) + f.flush() + monkeypatch.setattr(const, 'SG_CAL_FILE_PATH', Path(f.name)) + yield f + + +@pytest.fixture +def tempcal_file(monkeypatch: pytest.MonkeyPatch) -> FileIO: + f = NamedTemporaryFile() + f.writelines([ + f'{s}\n'.encode() + for s in [ + 'Black, 39,40', + 'Black, 46,48', + 'Black, 54,55', + 'Black, 60,62', + 'Black, 68,70', + 'Black, 76,76', + ]]) + f.flush() + monkeypatch.setattr(const, 'TEMP_CAL_FILE_PATH', Path(f.name)) + yield f + + +@pytest.fixture +def tempfiles(monkeypatch: pytest.MonkeyPatch, + sgcal_file: FileIO, + tempcal_file: FileIO, + devices_file: FileIO, + config_dir: TemporaryDirectory): + return diff --git a/test/test_parser.py b/test/test_parser.py index 8a90297..0c4ead9 100644 --- a/test/test_parser.py +++ b/test/test_parser.py @@ -2,136 +2,65 @@ Tests brewblox_tilt.parser """ -import json -from pathlib import Path -from tempfile import NamedTemporaryFile, TemporaryDirectory - import pytest -from brewblox_tilt import const, parser +from brewblox_tilt import const, mqtt, parser, stored TESTED = parser.__name__ -RED_MAC = 'AA7F97FC141E' -PURPLE_MAC = 'BB7F97FC141E' -BLACK_MAC = 'DD7F97FC141E' - - -@pytest.fixture -def m_sg_file(): - f = NamedTemporaryFile() - f.writelines([ - f'{s}\n'.encode() - for s in [ - 'Black, 1.000, 2.001', - 'Black, 1.001, 2.002', - 'Black, 1.002, 2.003', - 'BLACK, 1.003, 2.004', - 'Black, 1, Many', - 'Black, Few, 2.005', - '' - '"Ferment 1 red", 1.000, 3.010', - '"Ferment 1 red", 1.001, 3.011', - '"Ferment 1 red", 1.002, 3.012', - '"Ferment 1 red", 1.003, 3.013', - '"Ferment 1 red", 1.004, 3.014', - ]]) - f.flush() - yield f - - -@pytest.fixture -def m_temp_file(): - f = NamedTemporaryFile() - f.writelines([ - f'{s}\n'.encode() - for s in [ - 'Black, 39,40', - 'Black, 46,48', - 'Black, 54,55', - 'Black, 60,62', - 'Black, 68,70', - 'Black, 76,76', - ]]) - f.flush() - yield f - - -@pytest.fixture -def m_devices_file(): - f = NamedTemporaryFile() - f.write(json.dumps({'names': {BLACK_MAC: 'Black'}}).encode()) - f.flush() - yield f +@pytest.fixture(autouse=True) +def setup(tempfiles): + mqtt.setup() + stored.setup() + parser.setup() -@pytest.fixture -def m_config_dir(): - d = TemporaryDirectory() - yield d +def test_data_parser(tilt_macs: dict): + data_parser = parser.CV.get() + devices = stored.DEVICES.get() -@pytest.fixture -def m_files(mocker, m_sg_file, m_temp_file, m_devices_file, m_config_dir): - mocker.patch(TESTED + '.const.CONFIG_DIR', Path(m_config_dir.name)) - mocker.patch(TESTED + '.const.SG_CAL_FILE_PATH', Path(m_sg_file.name)) - mocker.patch(TESTED + '.const.TEMP_CAL_FILE_PATH', Path(m_temp_file.name)) - mocker.patch(TESTED + '.const.DEVICES_FILE_PATH', Path(m_devices_file.name)) + red_mac = tilt_macs['red'] + black_mac = tilt_macs['black'] + purple_mac = tilt_macs['purple'] - -def test_calibrator(m_sg_file): - calibrator = parser.Calibrator(m_sg_file.name) - assert 'black' in calibrator.cal_polys - assert 'ferment 1 red' in calibrator.cal_polys - assert calibrator.cal_polys['black'].order == 3 - - cal_black_v = calibrator.calibrated_value(['Dummy', 'Black'], 1.002, 3) - assert cal_black_v == pytest.approx(2, 0.1) - - cal_red_v = calibrator.calibrated_value(['Ferment 1 red'], 1.002, 3) - assert cal_red_v == pytest.approx(3, 0.1) - - assert calibrator.calibrated_value(['Dummy'], 1.002, 3) is None - - -def test_data_parser(app, m_files): red_uuid = next((k for k, v in const.TILT_UUID_COLORS.items() if v == 'Red')) purple_uuid = next((k for k, v in const.TILT_UUID_COLORS.items() if v == 'Purple')) black_uuid = next((k for k, v in const.TILT_UUID_COLORS.items() if v == 'Black')) - data_parser = parser.EventDataParser(app) - data_parser.apply_custom_names({RED_MAC: 'Ferment 1 red'}) + + devices.apply_custom_names({red_mac: 'Ferment 1 red'}) messages = data_parser.parse([ # Valid red - SG calibration data - parser.TiltEvent(mac=RED_MAC, + parser.TiltEvent(mac=red_mac, uuid=red_uuid, major=68, # temp F minor=1.002*1000, # raw SG, txpower=0, rssi=-80), # Valid black - SG and temp calibration data - parser.TiltEvent(mac=BLACK_MAC, + parser.TiltEvent(mac=black_mac, uuid=black_uuid, major=68, # temp F minor=1.002*1000, # raw SG, txpower=0, rssi=-80), # Invalid: out of bounds SG - parser.TiltEvent(mac=RED_MAC, + parser.TiltEvent(mac=red_mac, uuid=red_uuid, major=68, # temp F minor=1.002*1000000, # raw SG, txpower=0, rssi=-80), # Invalid: invalid UUID - parser.TiltEvent(mac=RED_MAC, + parser.TiltEvent(mac=red_mac, uuid='', major=68, # temp F minor=1.002*1000000, # raw SG, txpower=0, rssi=-80), # Valid purple - no calibration data - parser.TiltEvent(mac=PURPLE_MAC, + parser.TiltEvent(mac=purple_mac, uuid=purple_uuid, major=68, # temp F minor=1.002*1000, # raw SG, @@ -143,7 +72,7 @@ def test_data_parser(app, m_files): # Red msg = messages[0] - assert msg.mac == RED_MAC + assert msg.mac == red_mac assert msg.color == 'Red' assert msg.name == 'Ferment 1 red' assert msg.data == { @@ -159,7 +88,7 @@ def test_data_parser(app, m_files): # Black msg = messages[1] - assert msg.mac == BLACK_MAC + assert msg.mac == black_mac assert msg.color == 'Black' assert msg.name == 'Black' print(msg.data) @@ -178,9 +107,9 @@ def test_data_parser(app, m_files): # Purple msg = messages[2] - assert msg.mac == PURPLE_MAC + assert msg.mac == purple_mac assert msg.color == 'Purple' - assert msg.name == 'Purple' + assert msg.name == 'Ferment 1 Tilt' assert msg.data == { 'temperature[degF]': pytest.approx(68), 'temperature[degC]': pytest.approx((68-32)*5/9, 0.01), diff --git a/test/test_stored.py b/test/test_stored.py index c9f7857..90cce61 100644 --- a/test/test_stored.py +++ b/test/test_stored.py @@ -2,32 +2,34 @@ Tests brewblox_tilt.config """ import json +from io import FileIO from tempfile import NamedTemporaryFile import pytest +from pytest_mock import MockerFixture -from brewblox_tilt import stored +from brewblox_tilt import mqtt, stored TESTED = stored.__name__ +@pytest.fixture(autouse=True) +def setup(tempfiles): + mqtt.setup() + stored.setup() + + def default_names(): + # Matches devices from conftest return { - 'DD7F97FC141E': 'Purple', - 'EE7F97FC141E': 'Ferment 1 tilt', + 'AA7F97FC141E': 'Red', + 'DD7F97FC141E': 'Black', + 'BB7F97FC141E': 'Ferment 1 Tilt', } -@pytest.fixture -def m_file(): - f = NamedTemporaryFile() - f.write(json.dumps({'names': default_names()}).encode()) - f.flush() - return f - - -def test_load(m_file): - registry = stored.DeviceConfig(m_file.name) +def test_load(): + registry = stored.DEVICES.get() assert registry.names == default_names() @@ -53,9 +55,9 @@ def test_sanitize(): } -def test_lookup(m_file): - registry = stored.DeviceConfig(m_file.name) - assert registry.lookup('DD7F97FC141E', '') == 'Purple' +def test_lookup(): + registry = stored.DEVICES.get() + assert registry.lookup('DD7F97FC141E', '') == 'Black' assert registry.lookup('AA7F97FC141E', 'Red') == 'Red' assert registry.lookup('AB7F97FC141E', 'Red') == 'Red-2' assert registry.lookup('AC7F97FC141E', 'Red') == 'Red-3' @@ -73,8 +75,8 @@ def test_lookup(m_file): registry.lookup('Dummy', 'Black') -def test_apply_custom_names(m_file): - registry = stored.DeviceConfig(m_file.name) +def test_apply_custom_names(): + registry = stored.DEVICES.get() registry.apply_custom_names({ 'AA7F97FC141E': 'Red', 'BB7F97FC141E': 'Red', # Duplicate name @@ -93,22 +95,39 @@ def test_apply_custom_names(m_file): assert registry.changed -def test_commit(m_file, mocker): - registry = stored.DeviceConfig(m_file.name) +def test_autocommit(devices_file: FileIO, mocker: MockerFixture): + registry = stored.DeviceConfig(devices_file.name) mocker.patch.object(registry, 'yaml', wraps=registry.yaml) - registry.lookup('AA7F97FC141E', 'Red') - # Changes are not yet committed to file - registry2 = stored.DeviceConfig(m_file.name) - assert registry2.names == default_names() + with registry.autocommit(): + # add new item + registry.lookup('FF7F97FC141E', 'Red 2') + + # Changes are not yet committed to file + registry2 = stored.DeviceConfig(devices_file.name) + assert registry2.names == default_names() + assert registry.yaml.dump.call_count == 0 - registry.commit() - registry.commit() assert registry.yaml.dump.call_count == 1 # Changes are committed and present in file - registry3 = stored.DeviceConfig(m_file.name) + registry3 = stored.DeviceConfig(devices_file.name) assert registry3.names == { **default_names(), - 'AA7F97FC141E': 'Red', + 'FF7F97FC141E': 'Red 2', } + + +def test_calibrator(): + calibrator = stored.SG_CAL.get() + assert 'black' in calibrator.cal_polys + assert 'ferment 1 red' in calibrator.cal_polys + assert calibrator.cal_polys['black'].order == 3 + + cal_black_v = calibrator.calibrated_value(['Dummy', 'Black'], 1.002, 3) + assert cal_black_v == pytest.approx(2, 0.1) + + cal_red_v = calibrator.calibrated_value(['Ferment 1 red'], 1.002, 3) + assert cal_red_v == pytest.approx(3, 0.1) + + assert calibrator.calibrated_value(['Dummy'], 1.002, 3) is None