Merge pull request #54 from DerOetzi/import-historic-data-from-monito…

…ring

Import old historic data csv from monitoring site to influxdb

import_historic.py

@@ -0,0 +1,105 @@
+import asyncio
+from datetime import timezone
+from pandas import read_csv, to_datetime
+
+from solaredge2mqtt.models import (
+    BatteryPowerflow,
+    ConsumerPowerflow,
+    GridPowerflow,
+    InverterPowerflow,
+    Powerflow,
+)
+from solaredge2mqtt.service.influxdb import InfluxDB, Point
+from solaredge2mqtt.settings import LOCAL_TZ, service_settings
+
+settings = service_settings()
+STRINGS = 2
+
+
+async def main():
+    data = read_csv("data.csv")
+    data["time"] = to_datetime(data["time"], format="%d.%m.%Y %H:%M").dt.tz_localize(
+        LOCAL_TZ
+    )
+
+    for i in range(1, STRINGS + 1):
+        data[f"string{i}"] = data[f"string{i}"].fillna(0)
+
+    data["evcharger"] = data["evcharger"].fillna(0)
+
+    data.info()
+
+    points = []
+
+    columns = data.columns.to_list()
+
+    for _, row in data.iterrows():
+
+        grid = GridPowerflow(
+            power=round(row["grid_delivery"] - row["grid_consumption"])
+        )
+
+        if "battery_charge" in columns and "battery_discharge" in columns:
+            battery = BatteryPowerflow(
+                power=round(row["battery_charge"] - row["battery_discharge"])
+            )
+        else:
+            battery = BatteryPowerflow(power=0)
+
+        pv_production = 0.0
+        for i in range(1, STRINGS + 1):
+            pv_production += row[f"string{i}"]
+
+        dc_power = pv_production - battery.power
+
+        inverter = InverterPowerflow(
+            power=round(row["inverter_production"]),
+            dc_power=round(dc_power),
+            battery_discharge=battery.discharge,
+        )
+
+        if "evcharger" in columns:
+            evcharger = round(row["evcharger"])
+        else:
+            evcharger = 0
+
+        consumer = ConsumerPowerflow(inverter=inverter, grid=grid, evcharger=evcharger)
+
+        pv_production = round(pv_production)
+
+        powerflow = Powerflow(
+            pv_production=pv_production,
+            inverter=inverter,
+            grid=grid,
+            battery=battery,
+            consumer=consumer,
+        )
+
+        print(
+            f"Time: {row['time']}, PV: {pv_production} W, Inverter: {inverter.power} W, "
+            + f"House: {consumer.house} W, Grid: {grid.power} W, "
+            + f"Battery: {battery.power} W, Wallbox: {consumer.evcharger} W"
+        )
+
+        point = powerflow.prepare_point("powerflow_import")
+        point.tag("agg_type", "mean")
+        point.time(row["time"].astimezone(timezone.utc))
+        points.append(point)
+
+        point = powerflow.prepare_point_energy("energy_import", settings.prices)
+        point.time(row["time"].astimezone(timezone.utc))
+        points.append(point)
+
+        if "battery_soc" in columns:
+            point = Point("battery_import")
+            point.field("state_of_charge", round(row["battery_soc"], 2))
+            point.time(row["time"].astimezone(timezone.utc))
+            point.tag("agg_type", "mean")
+            points.append(point)
+
+    influxdb = InfluxDB(settings.influxdb, settings.prices)
+    await influxdb.write_points_async(points)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

solaredge2mqtt/flux/aggregate.flux

@@ -32,8 +32,6 @@ power
     |> map(fn: (r) => ({r with _time: date.truncate(t: date.sub(from: r._time, d: 1s), unit: 1h)}))
     |> to(bucket: bucket)
 
-needed = ["pv_production", "consumer_used_production", "grid_delivery"]
-
 energy = power
     |> aggregateWindow(
         every: 1h,
@@ -49,7 +47,7 @@ energy = power
 energy
     |> filter(fn: (r) => r._field == "consumer_used_production")
     |> map(fn: (r) => ({r with _value: r._value * {{PRICE_IN}}}))
-    |> set(key: "_field", value: "money_savings")
+    |> set(key: "_field", value: "money_saved")
     |> to(bucket: bucket)
     |> map(fn: (r) => ({r with _value: {{PRICE_IN}}}))
     |> set(key: "_field", value: "money_price_in")
@@ -58,12 +56,20 @@ energy
 energy
     |> filter(fn: (r) => r._field == "grid_delivery")
     |> map(fn: (r) => ({r with _value: r._value * {{PRICE_OUT}}}))
-    |> set(key: "_field", value: "money_earnings")
+    |> set(key: "_field", value: "money_delivered")
     |> to(bucket: bucket)
     |> map(fn: (r) => ({r with _value: {{PRICE_OUT}}}))
     |> set(key: "_field", value: "money_price_out")
     |> to(bucket: bucket)
 
+energy
+    |> filter(fn: (r) => r._field == "grid_consumption")
+    |> map(fn: (r) => ({r with _value: r._value * {{PRICE_IN}}}))
+    |> set(key: "_field", value: "money_consumed")
+    |> to(bucket: bucket)
+
+
+
 battery =
     from(bucket: bucket)
         |> range(start: startTime)

solaredge2mqtt/flux/fixes/import-historic-monitoring_data_1.0.8.flux

@@ -0,0 +1,47 @@
+import "date"
+
+
+startTime = date.sub(from: date.truncate(t: now(), unit: 1y), d: 1y)
+stopTime = date.add(to: date.truncate(t: now(), unit: 1h), d: 1d)
+
+bucket = "solaredgedev"
+
+from(bucket: bucket)
+    |> range(start: startTime, stop: stopTime)
+    |> filter(fn: (r) => r["_measurement"] == "powerflow")
+    |> set(key: "_measurement", value: "powerflow_import")
+    |> to(bucket: bucket)
+
+from(bucket: bucket)
+    |> range(start: startTime, stop: stopTime)
+    |> filter(fn: (r) => r["_measurement"] == "energy")
+    |> set(key: "_measurement", value: "energy_import")
+    |> to(bucket: bucket)
+
+from(bucket: bucket)
+    |> range(start: startTime, stop: stopTime)
+    |> filter(fn: (r) => r["_measurement"] == "battery")
+    |> set(key: "_measurement", value: "battery_import")
+    |> to(bucket: bucket)
+
+exclude = ["inverter_power", "grid_power", "battery_power"]
+
+from(bucket: bucket)
+    |> range(start: startTime, stop: stopTime)
+    |> filter(fn: (r) => r["_measurement"] == "powerflow_import")
+    |> filter(fn: (r) => not contains(value: r._field, set: exclude))
+    |> set(key: "_measurement", value: "powerflow")
+    |> to(bucket: bucket)
+
+from(bucket: bucket)
+    |> range(start: startTime, stop: stopTime)
+    |> filter(fn: (r) => r["_measurement"] == "energy_import")
+    |> filter(fn: (r) => not contains(value: r._field, set: exclude))
+    |> set(key: "_measurement", value: "energy")
+    |> to(bucket: bucket)
+
+from(bucket: bucket)
+    |> range(start: startTime, stop: stopTime)
+    |> filter(fn: (r) => r["_measurement"] == "battery_import")
+    |> set(key: "_measurement", value: "battery")
+    |> to(bucket: bucket)

solaredge2mqtt/flux/fixes/refactor_money_fields_1.0.8.flux

@@ -0,0 +1,29 @@
+import "date"
+
+startTime = date.sub(from: date.truncate(t: now(), unit: 1y), d: 1y)
+stopTime = date.add(to: date.truncate(t: now(), unit: 1h), d: 1d)
+
+bucket = "solaredgedev"
+price_in = 0.3644
+
+from(bucket: bucket)
+    |> range(start: startTime, stop: stopTime)
+    |> filter(fn: (r) => r["_measurement"] == "energy")
+    |> filter(fn: (r) => r["_field"] == "money_earnings")
+    |> set(key: "_field", value: "money_delivered")
+    |> to(bucket: bucket)
+
+from(bucket: bucket)
+    |> range(start: startTime, stop: stopTime)
+    |> filter(fn: (r) => r["_measurement"] == "energy")
+    |> filter(fn: (r) => r["_field"] == "money_savings")
+    |> set(key: "_field", value: "money_saved")
+    |> to(bucket: bucket)
+
+from(bucket: bucket)
+    |> range(start: startTime, stop: stopTime)
+    |> filter(fn: (r) => r["_measurement"] == "energy")
+    |> filter(fn: (r) => r["_field"] == "grid_consumption")
+    |> map(fn: (r) => ({r with _value: r._value * price_in}))
+    |> set(key: "_field", value: "money_consumed")
+    |> to(bucket: bucket)

solaredge2mqtt/models/__init__.py

@@ -13,7 +13,13 @@
     LogicalModule,
     LogicalString,
 )
-from solaredge2mqtt.models.powerflow import Powerflow
+from solaredge2mqtt.models.powerflow import (
+    BatteryPowerflow,
+    ConsumerPowerflow,
+    GridPowerflow,
+    InverterPowerflow,
+    Powerflow,
+)
 from solaredge2mqtt.models.wallbox import WallboxAPI
 from solaredge2mqtt.models.weather import (
     OpenWeatherMapBaseData,

solaredge2mqtt/models/historic.py

@@ -76,11 +76,22 @@ def self_sufficiency_rates(self) -> SelfSufficiencyRate:
 
 
 class HistoricMoney(Solaredge2MQTTBaseModel):
-    earnings: float
-    savings: float
+    delivered: float
+    saved: float
+    consumed: float
     price_in: float = Field(exclude=True)
     price_out: float = Field(exclude=True)
 
+    @computed_field
+    @property
+    def balance_grid(self) -> float:
+        return round(self.delivered - self.consumed, 3)
+
+    @computed_field
+    @property
+    def balance_total(self) -> float:
+        return round(self.balance_grid + self.saved, 3)
+
 
 class HistoricInfo(Solaredge2MQTTBaseModel):
     period: HistoricPeriod
@@ -109,6 +120,8 @@ class HistoricPeriod(EnumModel):
     THIS_MONTH = "this_month", "1mo", HistoricQuery.ACTUAL
     LAST_MONTH = "last_month", "1mo", HistoricQuery.LAST
     THIS_YEAR = "this_year", "1y", HistoricQuery.ACTUAL
+    LAST_YEAR = "last_year", "1y", HistoricQuery.LAST
+    LIFETIME = "lifetime", "99y", HistoricQuery.ACTUAL
 
     def __init__(self, topic: str, unit: str, query: HistoricQuery) -> None:
         self._topic: str = topic