Skip to content

Commit

Permalink
Merge pull request openWB#1023 from LKuemmel/smarthome_logging
Browse files Browse the repository at this point in the history 
Smarthome logging
  • Loading branch information
@LKuemmel
LKuemmel authored Jul 6, 2023
2 parents 0b746a6 + 468ecce commit fe46a4f
Show file tree
Hide file tree
Showing 3 changed files with 131 additions and 18 deletions.
82 changes: 75 additions & 7 deletions packages/helpermodules/measurement_log.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,22 @@
import logging
import pathlib
from pathlib import Path
from typing import Dict, List
import re
import string
from paho.mqtt.client import Client as MqttClient, MQTTMessage
from typing import Dict, List, Tuple

from control import data
from helpermodules.broker import InternalBrokerClient
from helpermodules.pub import Pub
from helpermodules import timecheck
from control.bat import Bat
from control.chargepoint.chargepoint import Chargepoint
from control.counter import Counter
from control.ev import Ev
from control.pv import Pv
from helpermodules.utils.topic_parser import decode_payload, get_index
from modules.common.utils.component_parser import get_component_name_by_id

log = logging.getLogger(__name__)

Expand Down Expand Up @@ -78,9 +84,10 @@ def save_log(folder):
}
... (dynamisch, je nach konfigurierter Anzahl)
}
"smarthome_devices": {
"device1": {
"counter": Wh,
"sh": {
"sh1": {
"exported": Wh,
"imported": Wh,
wenn konfiguriert:
"temp1": int in °C,
"temp2": int in °C,
Expand All @@ -99,6 +106,11 @@ def save_log(folder):
'cp5': {'exported': 0, 'imported': 0},
'cp6': {'exported': 0, 'imported': 64}},
'pv': {'all': {'imported': 251}, 'pv1': {'imported': 247}}}
},
"names": {
"counter0": "Mein EVU-Zähler",
"bat2": "Mein toller Speicher",
...
}
}
Expand Down Expand Up @@ -165,14 +177,17 @@ def save_log(folder):
except Exception:
log.exception("Fehler im Werte-Logging-Modul für Speicher "+str(bat))

sh_dict, sh_names = LegacySmarthomeLogdata().update()

new_entry = {
"timestamp": current_timestamp,
"date": date,
"cp": cp_dict,
"ev": ev_dict,
"counter": counter_dict,
"pv": pv_dict,
"bat": bat_dict
"bat": bat_dict,
"sh": sh_dict
}

# json-Objekt in Datei einfügen
Expand All @@ -199,14 +214,15 @@ def save_log(folder):
entries = content["entries"]
entries.append(new_entry)
content["totals"] = get_totals(entries)
content["names"] = get_names(content["totals"], sh_names)
with open(filepath, "w") as jsonFile:
json.dump(content, jsonFile)
return content["totals"]


def get_totals(entries: List, sum_up_diffs: bool = False) -> Dict:
# beim Jahreslog werden die Summen aus den Monatssummen berechnet, bei allen anderen aus den absoluten Zählerwerten
totals: Dict[str, Dict] = {"cp": {}, "counter": {}, "pv": {}, "bat": {}}
totals: Dict[str, Dict] = {"cp": {}, "counter": {}, "pv": {}, "bat": {}, "sh": {}}
prev_entry: Dict = {}
for group in totals.keys():
for entry in entries:
Expand All @@ -218,7 +234,7 @@ def get_totals(entries: List, sum_up_diffs: bool = False) -> Dict:
totals[group][module] = {"exported": 0} if group == "pv" else {"imported": 0, "exported": 0}
else:
for key, value in entry[group][module].items():
if key != "soc":
if key != "soc" and "temp" not in key:
if sum_up_diffs:
value = (Decimal(str(value))
+ Decimal(str(totals[group][module][key])))
Expand All @@ -240,6 +256,23 @@ def get_totals(entries: List, sum_up_diffs: bool = False) -> Dict:
return totals


def get_names(totals: Dict, sh_names: Dict) -> Dict:
names = sh_names
for group in totals.items():
if group[0] == "sh":
continue
for entry in group[1]:
try:
if "cp" in entry:
names.update({entry: data.data.cp_data[entry].data.config.name})
elif "all" != entry:
id = entry.strip(string.ascii_letters)
names.update({entry: get_component_name_by_id(int(id))})
except (ValueError, KeyError):
names.update({entry: entry})
return names


def get_daily_log(date: str):
try:
with open(str(Path(__file__).resolve().parents[2] / "data"/"daily_log"/(date+".json")), "r") as jsonFile:
Expand Down Expand Up @@ -314,3 +347,38 @@ def update_exported(daily_exported: float) -> None:
if module == "cp" and m == "all":
module_data = data.data.cp_all_data
update_imported_exported(totals[module][m]["imported"], totals[module][m]["exported"])


class LegacySmarthomeLogdata:
def __init__(self) -> None:
self.all_received_topics: Dict = {}

def update(self) -> Tuple[Dict, Dict]:
sh_dict: Dict = {}
sh_names: Dict = {}
try:
InternalBrokerClient("smarthome-logging", self.on_connect, self.on_message).start_finite_loop()
for topic, payload in self.all_received_topics.items():
if re.search("openWB/LegacySmartHome/config/get/Devices/[1-9]/device_configured", topic) is not None:
if decode_payload(payload) == 1:
index = get_index(topic)
sh_dict.update({f"sh{index}": {}})
for topic, payload in self.all_received_topics.items():
if f"openWB/LegacySmartHome/Devices/{index}/Wh" == topic:
sh_dict[f"sh{index}"].update({"imported": decode_payload(payload), "exported": 0})
for sensor_id in range(0, 3):
if f"openWB/LegacySmartHome/Devices/{index}/TemperatureSensor{sensor_id}" == topic:
sh_dict[f"sh{index}"].update({f"temp{sensor_id}": decode_payload(payload)})
for topic, payload in self.all_received_topics.items():
if f"openWB/LegacySmartHome/config/get/Devices/{index}/device_name" == topic:
sh_names.update({f"sh{index}": decode_payload(payload)})
except Exception:
log.exception("Fehler im Werte-Logging-Modul für Smarthome")
finally:
return sh_dict, sh_names

def on_connect(self, client: MqttClient, userdata, flags: dict, rc: int):
client.subscribe("openWB/LegacySmartHome/#", 2)

def on_message(self, client: MqttClient, userdata, msg: MQTTMessage):
self.all_received_topics.update({msg.topic: msg.payload})
44 changes: 35 additions & 9 deletions packages/helpermodules/measurement_log_test.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import threading
from unittest.mock import Mock
import pytest
from control.chargepoint import chargepoint

Expand All @@ -8,14 +9,6 @@
from control import data


def test_get_totals():
# execution
totals = measurement_log.get_totals(SAMPLE)

# evaluation
assert totals == TOTALS


@pytest.fixture(autouse=True)
def data_module() -> None:
data.data_init(threading.Event())
Expand All @@ -27,6 +20,25 @@ def data_module() -> None:
data.data.pv_data.update({"all": pv_all.PvAll(), "pv1": pv.Pv(1)})


def test_get_totals(monkeypatch):
# execution
totals = measurement_log.get_totals(SAMPLE)

# evaluation
assert totals == TOTALS


def test_get_names(monkeypatch):
# setup
component_names_mock = Mock(side_effect=["Speicher", "Zähler", "Wechselrichter"])
monkeypatch.setattr(measurement_log, "get_component_name_by_id", component_names_mock)
# execution
names = measurement_log.get_names(TOTALS, {"sh1": "Smarthome1"})

# evaluation
assert names == NAMES


def test_get_daily_yields(mock_pub):
# setup and execution
[measurement_log.update_module_yields(type, TOTALS) for type in ("bat", "counter", "cp", "pv")]
Expand Down Expand Up @@ -72,6 +84,7 @@ def test_get_daily_yields(mock_pub):
'date': '13:41',
'ev': {'ev0': {'soc': 0}},
'pv': {'all': {'exported': 88}, 'pv1': {'exported': 92}},
"sh": {},
'timestamp': 1654861269},
{'bat': {'all': {'exported': 0, 'imported': 146.108, 'soc': 53},
'bat2': {'exported': 0, 'imported': 149.099, 'soc': 53}},
Expand All @@ -83,6 +96,7 @@ def test_get_daily_yields(mock_pub):
'date': '13:46',
'ev': {'ev0': {'soc': 4}},
'pv': {'all': {'exported': 214}, 'pv1': {'exported': 214}},
"sh": {"sh1": {"temp0": 300, "temp1": 300, "temp2": 300, "imported": 100, "exported": 0}},
'timestamp': 1654861569},
{'bat': {'all': {'exported': 0, 'imported': 234.308, 'soc': 55},
'bat2': {'exported': 0, 'imported': 234.308, 'soc': 55}},
Expand All @@ -96,6 +110,7 @@ def test_get_daily_yields(mock_pub):
'date': '13:51',
'ev': {'ev0': {'soc': 6}},
'pv': {'all': {'exported': 339}, 'pv1': {'exported': 339}},
"sh": {"sh1": {"temp0": 300, "temp1": 300, "temp2": 300, "imported": 200, "exported": 0}},
'timestamp': 1654861869},
{'bat': {'all': {'exported': 0, 'imported': 234.308, 'soc': 55},
'bat2': {'exported': 0, 'imported': 234.308, 'soc': 55}},
Expand All @@ -107,6 +122,7 @@ def test_get_daily_yields(mock_pub):
'date': '13:51',
'ev': {'ev0': {'soc': 6}},
'pv': {'all': {'exported': 339}, 'pv1': {'exported': 339}},
"sh": {"sh1": {"temp0": 300, "temp1": 300, "temp2": 300, "imported": 400, "exported": 0}},
'timestamp': 1654862069}]

TOTALS = {'bat': {'all': {'exported': 0, 'imported': 175.534},
Expand All @@ -117,4 +133,14 @@ def test_get_daily_yields(mock_pub):
'cp4': {'exported': 0, 'imported': 85},
'cp5': {'exported': 0, 'imported': 0},
'cp6': {'exported': 0, 'imported': 2}},
'pv': {'all': {'exported': 251}, 'pv1': {'exported': 247}}}
'pv': {'all': {'exported': 251}, 'pv1': {'exported': 247}},
"sh": {"sh1": {"imported": 300, "exported": 0}}}

NAMES = {'bat2': "Speicher",
'counter0': "Zähler",
'cp3': "cp3",
'cp4': "Standard-Ladepunkt",
'cp5': "Standard-Ladepunkt",
'cp6': "Standard-Ladepunkt",
'pv1': "Wechselrichter",
"sh1": "Smarthome1"}
23 changes: 21 additions & 2 deletions packages/helpermodules/update_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@


class UpdateConfig:
DATASTORE_VERSION = 15
DATASTORE_VERSION = 16
valid_topic = [
"^openWB/bat/config/configured$",
"^openWB/bat/set/charging_power_left$",
Expand Down Expand Up @@ -499,7 +499,7 @@ def __solve_breaking_changes(self) -> None:
try:
getattr(self, f"upgrade_datastore_{version}")()
except AttributeError:
log.error("missing upgrade function! $version$")
log.error(f"missing upgrade function! {version}")

def upgrade_datastore_0(self) -> None:
# prevent_switch_stop auf zwei Einstellungen prevent_phase_switch und prevent_charge_stop aufteilen
Expand Down Expand Up @@ -759,3 +759,22 @@ def upgrade_datastore_14(self) -> None:
payload.configuration.pop("ip_adress")
Pub().pub(topic.replace("openWB/", "openWB/set/"), payload)
Pub().pub("openWB/system/datastore_version", 15)

def upgrade_datastore_15(self) -> None:
files = glob.glob("/var/www/html/openWB/data/daily_log/*")
files.extend(glob.glob("/var/www/html/openWB/data/monthly_log/*"))
for file in files:
with open(file, "r+") as jsonFile:
try:
content = json.load(jsonFile)
for e in content["entries"]:
e.update({"sh": {}})
content["totals"].update({"sh": {}})
content["names"] = measurement_log.get_names(content["totals"], {})
jsonFile.seek(0)
json.dump(content, jsonFile)
jsonFile.truncate()
log.debug(f"Format der Logdatei {file} aktualisiert.")
except Exception:
log.exception(f"Logfile {file} konnte nicht konvertiert werden.")
Pub().pub("openWB/system/datastore_version", 16)

0 comments on commit fe46a4f

Please sign in to comment.