From 9bb0151f1fdf8683ac8a34e538d4538794edd0f4 Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 10:27:52 -0700 Subject: [PATCH 01/17] Initial local monitoring --- .gitignore | 3 + .vscode/settings.json | 7 ++ Dockerfile | 4 + config.py | 22 +++++ docker-compose.yml | 15 ++++ interfaces/carbon_api.py | 23 ++++++ main.py | 31 +++++++ modules/__init__.py | 0 modules/database.py | 30 +++++++ modules/electricitymaps_api.py | 97 ++++++++++++++++++++++ modules/grafana_display.py | 0 modules/kasa_monitor.py | 74 +++++++++++++++++ modules/watttime_api.py | 0 pytest.ini | 5 ++ requirements.txt | 7 ++ sql/init.sql | 7 ++ tests/test_database.py | 132 ++++++++++++++++++++++++++++++ tests/test_electricitymaps_api.py | 46 +++++++++++ tests/test_kasa_monitor.py | 95 +++++++++++++++++++++ 19 files changed, 598 insertions(+) create mode 100644 .vscode/settings.json create mode 100644 Dockerfile create mode 100644 config.py create mode 100644 docker-compose.yml create mode 100644 interfaces/carbon_api.py create mode 100644 main.py create mode 100644 modules/__init__.py create mode 100644 modules/database.py create mode 100644 modules/electricitymaps_api.py create mode 100644 modules/grafana_display.py create mode 100644 modules/kasa_monitor.py create mode 100644 modules/watttime_api.py create mode 100644 pytest.ini create mode 100644 requirements.txt create mode 100644 sql/init.sql create mode 100644 tests/test_database.py create mode 100644 tests/test_electricitymaps_api.py create mode 100644 tests/test_kasa_monitor.py diff --git a/.gitignore b/.gitignore index 68bc17f..e2360c8 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,6 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +#project specific ignores +em_cache.json \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9b38853 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "python.testing.pytestArgs": [ + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..99962cf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,4 @@ +# Use an official Python runtime as a parent image +FROM python:3.12-slim-bookworm + +RUN apt-get update && apt-get install -y software-properties-common postgresql postgresql-contrib diff --git a/config.py b/config.py new file mode 100644 index 0000000..49270c1 --- /dev/null +++ b/config.py @@ -0,0 +1,22 @@ +import os +# Configuration for Kasa + +# Configuration for Grafana +GRAFANA_URL = "your_grafana_url" +GRAFANA_API_KEY = "your_grafana_api_key" + +# Configuration for General API +API_KEY = "your_api_key" +API_TYPE = "electricitymaps" # or "watttime" + +# Update interval in seconds +UPDATE_INTERVAL_SEC = 15 # second interval to update + +# database connection information +db_config = { + "user": os.getenv("DB_USER"), + "password": os.getenv("DB_PASSWORD"), + "database": os.getenv("DB_NAME"), + "host": os.getenv("DB_HOST"), + "port": os.getenv("DB_PORT"), +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1d56aa7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3.1' + +services: + db: + image: postgres + restart: always + environment: + POSTGRES_DB: ${DB_NAME} + POSTGRES_PASSWORD: ${DB_PASSWORD} + POSTGRES_USER: ${DB_USER} + volumes: + - ./sql:/docker-entrypoint-initdb.d + ports: + - ${DB_PORT}:5432 + \ No newline at end of file diff --git a/interfaces/carbon_api.py b/interfaces/carbon_api.py new file mode 100644 index 0000000..2082fb6 --- /dev/null +++ b/interfaces/carbon_api.py @@ -0,0 +1,23 @@ +from abc import ABC, abstractmethod + +class CarbonAPI(ABC): + @abstractmethod + async def get_co2_by_latlon(self, lat: float, lon: float) -> float: + """ + Get the current CO2 emission rate per kWh for a specific location, identified by latitude and longitude. + + :param lat: The latitude of the location. + :param lon: The longitude of the location. + :return: The current CO2 emission rate per kWh. + """ + pass + + @abstractmethod + async def get_co2_by_gridid(self, grid_id: str) -> float: + """ + Get the current CO2 emission rate per kWh for a specific location, identified by grid ID. + + :param grid_id: The ID of the grid. + :return: The current CO2 emission rate per kWh. + """ + pass \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..b21e9c3 --- /dev/null +++ b/main.py @@ -0,0 +1,31 @@ +import asyncio +import config +from modules import kasa_monitor, grafana_display, electricitymaps_api, watttime_api, database +from interfaces import carbon_api +import os +from dotenv import load_dotenv + +async def main(): + load_dotenv() + # Create instances of each module + kasa = kasa_monitor.KasaMonitor() + #grafana = grafana_display.GrafanaDisplay() + db = database.Database(config.db_config) + + # Monitor energy use continuously in a subthread to avoid blocking the main thread + energy_use_task = asyncio.create_task(kasa.monitor_energy_use_continuously(db, delay=config.UPDATE_INTERVAL_SEC)) + + while True: + if energy_use_task.done(): + print("The energy monitoring task has completed.") + else: + print("The energy monitoring task is still running.") + + # view the current database values + print(await db.read_usage()) + + # Wait for next update + await asyncio.sleep(config.UPDATE_INTERVAL_SEC) + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/modules/__init__.py b/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/database.py b/modules/database.py new file mode 100644 index 0000000..25c2738 --- /dev/null +++ b/modules/database.py @@ -0,0 +1,30 @@ +import asyncio +import asyncpg + +class Database: + def __init__(self, db_config): + self.db_config = db_config + self.conn = None + + async def write_usage(self, energy_usage): + self.conn = await asyncpg.connect(**self.db_config) + try: + await self.conn.execute( + "INSERT INTO energy_usage (device, timestamp, power, avg_mg_co2) VALUES ($1, $2, $3, $4)", + energy_usage["device"], energy_usage["timestamp"], energy_usage["power"], energy_usage["avg_mg_co2"] + ) + finally: + await self.close() + + async def read_usage(self): + self.conn = await asyncpg.connect(**self.db_config) + try: + result = await self.conn.fetch("SELECT * FROM energy_usage") + finally: + await self.close() + return result + + async def close(self): + if self.conn is not None and not self.conn.is_closed(): + await self.conn.close() + self.conn = None \ No newline at end of file diff --git a/modules/electricitymaps_api.py b/modules/electricitymaps_api.py new file mode 100644 index 0000000..0327793 --- /dev/null +++ b/modules/electricitymaps_api.py @@ -0,0 +1,97 @@ + +import aiohttp +from interfaces.carbon_api import CarbonAPI +import os +from datetime import datetime, timezone, timedelta +import json + +class ElectricityMapAPI(CarbonAPI): + BASE_URL = "https://api-access.electricitymaps.com/free-tier/carbon-intensity/latest" + CACHE_FILE = "em_cache.json" + + def __init__(self, co2_time_threshold_mins=120, clear_cache=False): + self.api_key = os.getenv("EM_API_KEY") + self.co2_time_threshold_mins = co2_time_threshold_mins + self.CACHE_EXPIRY = timedelta(int(os.getenv("EM_CACHE_EXPIRY_MINS"))) + if clear_cache: + self._clear_cache() + else: + self._load_cache() + + def _load_cache(self): + if os.path.exists(self.CACHE_FILE): + with open(self.CACHE_FILE, 'r') as f: + cache = json.load(f) + # Convert the keys from strings to dictionaries and the timestamps from strings to datetime objects + self.cache = {key: [value[0], datetime.fromisoformat(value[1])] for key, value in cache.items()} + else: + self.cache = {} + + def _save_cache(self): + # Convert the timestamps from datetime objects to strings + cache = {key: [value[0], value[1].isoformat()] for key, value in self.cache.items()} + with open(self.CACHE_FILE, 'w') as f: + json.dump(cache, f) + + def _clear_cache(self): + self.cache = {} + self._save_cache() + + async def get_co2_by_gridid(self, grid_id: str) -> float: + ''' + Get the current CO2 emission rate per kWh for a specific location, identified by grid ID. + Will use a local cache to store the data for 30 minutes to ease the load on the API. + + Args: + grid_id (str): The grid ID of the specific location. + + Returns: + float: The CO2 emission rate in gCO2/kWh if data is available and less than 120 minutes old, otherwise returns None. + ''' + params = {"zone": grid_id} + return await self._get_co2_data(params) + + async def get_co2_by_latlon(self, lat: float, lon: float) -> float: + ''' + Get the current CO2 emission rate per kWh for a specific location, identified by latitude and longitude. + Will use a local cache to store the data for 30 minutes to ease the load on the API. + + Args: + lat (float): The latitude of the location. + lon (float): The longitude of the location. + + Returns: + float: The CO2 emission rate in gCO2/kWh if data is available and less than 120 minutes old, otherwise returns None. + ''' + params = {"lat": lat, "lon": lon} + return await self._get_co2_data(params) + + async def _get_co2_data(self, params) -> float: + # Check if the data is in the cache and is less than 30 minutes old + # Convert the params dictionary to a string to use as a key + cache_key = '_'.join(str(v) for v in params.values()) + if cache_key in self.cache and datetime.now(timezone.utc) - self.cache[cache_key][1] < self.CACHE_EXPIRY: + return self.cache[cache_key][0] + + headers = {"auth-token": self.api_key} + co2_data = None + async with aiohttp.ClientSession() as session: + async with session.get(self.BASE_URL, headers=headers, params=params) as response: + data = await response.json() + co2_data = self._process_co2_data(data) + + # Store the data and the current time in the cache + self.cache[cache_key] = (co2_data, datetime.now(timezone.utc)) + self._save_cache() # Save the cache to the file + + return co2_data + + def _process_co2_data(self, data): + data_updated_at = datetime.strptime(data["updatedAt"], '%Y-%m-%dT%H:%M:%S.%fZ').replace(tzinfo=timezone.utc) + #verify its within the last minutes defined by co2_time_threshold + if abs((data_updated_at - datetime.now(timezone.utc)).total_seconds()) / 60 > self.co2_time_threshold_mins: + return None + return float(data["carbonIntensity"]) + + + \ No newline at end of file diff --git a/modules/grafana_display.py b/modules/grafana_display.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/kasa_monitor.py b/modules/kasa_monitor.py new file mode 100644 index 0000000..52277c4 --- /dev/null +++ b/modules/kasa_monitor.py @@ -0,0 +1,74 @@ +from kasa import Discover, SmartPlug +from modules.database import Database +from modules.electricitymaps_api import ElectricityMapAPI +import asyncio +import time, os +from datetime import datetime, timezone + +class KasaMonitor: + def __init__(self, local_lat=None, local_lon=None, local_grid_id=None, co2_api_provider="ElectricityMaps"): + self.devices = [] + self.lat = None + self.lon = None + self.grid_id = None + if (local_lat is None or local_lon is None) and local_grid_id is None: + #TODO use a geoip service to get the lat/lon + self.lat = os.getenv("LOCAL_LAT") + self.lon = os.getenv("LOCAL_LON") + elif local_grid_id is not None: + self.grid_id = local_grid_id + else: + raise ValueError("Must provide either local_lat/local_lon or local_grid_id") + + if co2_api_provider == "ElectricityMaps": + self.co2_api = ElectricityMapAPI() + else: + raise ValueError("co2_api_provider must be 'EM' until others are supported") + + async def discover_devices(self): + # Discover Kasa devices on the network + self.devices = await Discover.discover() + + async def monitor_energy_use_once(self): + energy_values = {} + + #assert discover_devices has been called + if len(self.devices) == 0: + await self.discover_devices() + + # Get energy use for each device + for addr, device in self.devices.items(): + await device.update() + if device.has_emeter: + emeter_realtime = device.emeter_realtime + energy_values[device.alias] = emeter_realtime["power"] + + return energy_values + + #get carbon data by either grid or lat/lon + async def _get_co2_data(self): + if self.grid_id is not None: + return await self.co2_api.get_co2_by_gridid(self.grid_id) + else: + return await self.co2_api.get_co2_by_latlon(self.lat, self.lon) + + async def monitor_energy_use_continuously(self, db, delay, timeout=None): + start_time = time.time() + + try: + while True: + energy_values = await self.monitor_energy_use_once() + + #Get average CO2 from API + co2 = await self._get_co2_data() + for device, power in energy_values.items(): + + energy_usage = {"device": device, "timestamp": datetime.now(timezone.utc), "power": power, "avg_mg_co2": co2} + await db.write_usage(energy_usage) + + if timeout is not None and time.time() - start_time >= timeout: + break + + time.sleep(delay) + finally: + await db.close() \ No newline at end of file diff --git a/modules/watttime_api.py b/modules/watttime_api.py new file mode 100644 index 0000000..e69de29 diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..6151f49 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +markers = + real_device: marks tests that require a real kasa device (deselect with '-m "not real_device"') + real_database: marks tests that require a real database (deselect with '-m "not real_database"') + real_em_api: marks tests that require access to the real electricity maps api (deselect with '-m "not real_em_api"') \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f477de2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +python-kasa==0.5.4 +pytest==7.4.0 +pytest-asyncio==0.21.1 +psycopg2-binary==2.9.9 +asyncpg==0.29.0 +pytz==2023.3.post1 +aiohttp==3.9.1 \ No newline at end of file diff --git a/sql/init.sql b/sql/init.sql new file mode 100644 index 0000000..28c0daa --- /dev/null +++ b/sql/init.sql @@ -0,0 +1,7 @@ +CREATE TABLE energy_usage ( + device VARCHAR(255) NOT NULL, + timestamp TIMESTAMPTZ NOT NULL, + power real NOT NULL, + avg_mg_co2 real, + PRIMARY KEY (device, timestamp) +); \ No newline at end of file diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 0000000..201301f --- /dev/null +++ b/tests/test_database.py @@ -0,0 +1,132 @@ +# FILEPATH: /C:/Users/scott/source/repos/kasa_carbon/tests/test_database.py + +import pytest +import asyncpg +from unittest.mock import patch, AsyncMock, Mock +from modules.database import Database +import config +from datetime import datetime, timezone +import pytz + +@pytest.fixture(autouse=True) +@pytest.mark.real_database +async def setup_method(): + # Your setup code here + db_config = config.db_config + conn = await asyncpg.connect(**db_config) + await conn.execute("DELETE FROM energy_usage") + await conn.close() + + yield # This is where the test will run + + db_config = config.db_config + conn = await asyncpg.connect(**db_config) + await conn.execute("DELETE FROM energy_usage") + await conn.close() + print("Tearing down") + +@pytest.mark.asyncio +async def test_write_usage(): + # Arrange + db_config = {"user": "test", "password": "test", "database": "test", "host": "localhost"} + db = Database(db_config) + energy_usage = {"device": "test_device", "timestamp": "2022-01-01 00:00:00", "power": 100, "avg_mg_co2": 50} + + with patch('asyncpg.connect', new_callable=AsyncMock) as mock_connect: + mock_connect.return_value.execute = AsyncMock() + mock_connect.return_value.is_closed = Mock(return_value=False) + mock_connect.return_value.close = AsyncMock() + + # Act + await db.write_usage(energy_usage) + + # Assert + mock_connect.assert_called_once_with(**db_config) + mock_connect.return_value.execute.assert_called_once_with( + "INSERT INTO energy_usage (device, timestamp, power, avg_mg_co2) VALUES ($1, $2, $3, $4)", + energy_usage["device"], energy_usage["timestamp"], energy_usage["power"], energy_usage["avg_mg_co2"] + ) + mock_connect.return_value.close.assert_called_once() + +@pytest.mark.asyncio +async def test_read_usage(): + # Arrange + db_config = {"user": "test", "password": "test", "database": "test", "host": "localhost"} + db = Database(db_config) + + # Act + with patch('asyncpg.connect', new_callable=AsyncMock) as mock_connect: + mock_connect.return_value.fetch = AsyncMock() + mock_connect.return_value.is_closed = Mock(return_value=False) + mock_connect.return_value.close = AsyncMock() + await db.read_usage() + + # Assert + mock_connect.assert_called_once_with(**db_config) + mock_connect.return_value.fetch.assert_called_once_with("SELECT * FROM energy_usage") + mock_connect.return_value.close.assert_called_once() + +@pytest.mark.asyncio +@pytest.mark.real_database +async def test_write_usage_realdb(): + # Arrange + db_config = config.db_config + db = Database(db_config) + timestamp_str = "2022-01-01 00:00:00" + timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S") + timestamp = pytz.utc.localize(timestamp) # Make timestamp timezone aware and in UTC timezone + energy_usage = {"device": "test_device", "timestamp": timestamp, "power": 100.0, "avg_mg_co2": 50.0} + + # Act + await db.write_usage(energy_usage) + + # Assert + conn = await asyncpg.connect(**db_config) + try: + result = await conn.fetchrow("SELECT * FROM energy_usage WHERE device = $1", energy_usage["device"]) + assert result is not None + assert result["device"] == energy_usage["device"] + assert result["timestamp"] == energy_usage["timestamp"] + assert result["power"] == energy_usage["power"] + assert result["avg_mg_co2"] == energy_usage["avg_mg_co2"] + finally: + await conn.execute("DELETE FROM energy_usage") + await conn.close() + +@pytest.mark.asyncio +@pytest.mark.real_database +async def test_read_usage_realdb(): + # Arrange + db_config = config.db_config + db = Database(db_config) + timestamp_str = "2022-01-01 00:00:00" + timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S") + timestamp = pytz.utc.localize(timestamp) # Make timestamp timezone aware and in UTC timezone + energy_usage = {"device": "test_device", "timestamp": timestamp, "power": 100.0, "avg_mg_co2": 50.0} + + conn = await asyncpg.connect(**db_config) + try: + await conn.execute( + "INSERT INTO energy_usage (device, timestamp, power, avg_mg_co2) VALUES ($1, $2, $3, $4)", + energy_usage["device"], energy_usage["timestamp"], energy_usage["power"], energy_usage["avg_mg_co2"] + ) + finally: + await conn.close() + + # Act + result = await db.read_usage() + + # Assert + assert len(result) > 0 + assert result[0]["device"] == energy_usage["device"] + # Convert the original timestamp to the client's timezone + timestamp_client_tz = energy_usage["timestamp"].astimezone(timezone.utc) + assert result[0]["timestamp"] == timestamp_client_tz + assert result[0]["power"] == energy_usage["power"] + assert result[0]["avg_mg_co2"] == energy_usage["avg_mg_co2"] + + conn = await asyncpg.connect(**db_config) + try: + await conn.execute("DELETE FROM energy_usage") + finally: + await conn.close() \ No newline at end of file diff --git a/tests/test_electricitymaps_api.py b/tests/test_electricitymaps_api.py new file mode 100644 index 0000000..70003f8 --- /dev/null +++ b/tests/test_electricitymaps_api.py @@ -0,0 +1,46 @@ +import pytest +from unittest.mock import patch, AsyncMock +from modules.electricitymaps_api import ElectricityMapAPI +from datetime import datetime, timedelta, timezone +import math + + +@pytest.mark.asyncio +@pytest.mark.real_em_api +async def test_get_co2_by_gridid_real_em_api(): + # Arrange + api = ElectricityMapAPI(clear_cache=True) + grid_id = "DE" + + # Act + result = await api.get_co2_by_gridid(grid_id) + + # Assert + assert isinstance(result, float) # The result should be a float + +@pytest.mark.asyncio +@pytest.mark.real_em_api +async def test_get_co2_by_latlon_real_em_api(): + # Arrange + api = ElectricityMapAPI(clear_cache=True) + lat, lon = 51.509865, -0.118092 + + # Act + result = await api.get_co2_by_latlon(lat, lon) + + # Assert + assert isinstance(result, float) # The result should be a float + +@pytest.mark.asyncio +@pytest.mark.real_em_api +async def test_get_co2_by_gridid_uses_cache(): + # Arrange + api = ElectricityMapAPI(clear_cache=True) + grid_id = "DE" + + # Act + result1 = await api.get_co2_by_gridid(grid_id) + result2 = await api.get_co2_by_gridid(grid_id) + + # Assert + assert result1 == result2 # The results should be the same \ No newline at end of file diff --git a/tests/test_kasa_monitor.py b/tests/test_kasa_monitor.py new file mode 100644 index 0000000..3a27d96 --- /dev/null +++ b/tests/test_kasa_monitor.py @@ -0,0 +1,95 @@ +import pytest +from unittest.mock import MagicMock, AsyncMock, Mock, patch +from modules.kasa_monitor import KasaMonitor +from modules.database import Database +import config +import asyncpg + +@pytest.mark.asyncio +async def test_discover_devices(): + # Arrange + with patch('kasa.Discover.discover') as mock_discover: + mock_discover.return_value = {} + kasa = KasaMonitor() + + # Act + await kasa.discover_devices() + + # Assert + mock_discover.assert_called_once() + +@pytest.mark.asyncio +async def test_monitor_energy_use(): + # Arrange + with patch('kasa.Discover.discover') as mock_discover: + mock_device = AsyncMock() + mock_device.has_emeter = True + mock_device.alias = "Device" + mock_device.emeter_realtime = {"power": 100} + mock_discover.return_value = {"192.168.0.1": mock_device} + kasa = KasaMonitor() + await kasa.discover_devices() + + # Act + energy_values = await kasa.monitor_energy_use_once() + + # Assert + assert energy_values == {"Device": 100} + + +@pytest.mark.asyncio +@pytest.mark.real_device +async def test_connect_to_real_device(): + # Arrange + kasa = KasaMonitor() + + # Act + await kasa.discover_devices() + + # Assert + assert len(kasa.devices) > 0, "No Kasa devices found on the network" + +@pytest.mark.asyncio +@pytest.mark.real_device +async def test_monitor_energy_use_real_device(): + # Arrange + kasa = KasaMonitor() + await kasa.discover_devices() + + # Act + energy_values = await kasa.monitor_energy_use_once() + + # Assert + assert energy_values, "No energy values were returned" + +@pytest.mark.asyncio +@pytest.mark.real_device +@pytest.mark.real_database +async def test_monitor_energy_use_continuously_integration(): + # Arrange + db_config = config.db_config + db = Database(db_config) + monitor = KasaMonitor() + + await monitor.discover_devices() + + # Act + # This will run the method for 60 seconds and then stop + await monitor.monitor_energy_use_continuously(db, delay=5, timeout=20) + + # Assert + # Check the database to ensure that the energy usage data was written correctly + result = await db.read_usage() + + assert len(result) > 0 + assert result[0]["device"] is not None + assert result[0]["timestamp"] is not None + assert result[0]["power"] is not None + assert result[0]["avg_mg_co2"] is not None + + #empty the db again + conn = await asyncpg.connect(**db_config) + try: + await conn.execute("DELETE FROM energy_usage") + finally: + await conn.close() \ No newline at end of file From bff32983ea9a71615049ee5d129b95b4fff446f0 Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 11:01:20 -0700 Subject: [PATCH 02/17] Updated monitor to monitor all plugs on a powerstrip --- modules/kasa_monitor.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/kasa_monitor.py b/modules/kasa_monitor.py index 52277c4..a65d6f2 100644 --- a/modules/kasa_monitor.py +++ b/modules/kasa_monitor.py @@ -1,4 +1,4 @@ -from kasa import Discover, SmartPlug +from kasa import Discover, SmartPlug, SmartStrip from modules.database import Database from modules.electricitymaps_api import ElectricityMapAPI import asyncio @@ -36,10 +36,14 @@ async def monitor_energy_use_once(self): if len(self.devices) == 0: await self.discover_devices() - # Get energy use for each device for addr, device in self.devices.items(): await device.update() - if device.has_emeter: + if isinstance(device, SmartStrip): + for i, plug in enumerate(device.children): + if plug.has_emeter: + emeter_realtime = plug.emeter_realtime + energy_values[f"{device.alias + '-' + plug.alias}"] = emeter_realtime["power"] + elif device.has_emeter: emeter_realtime = device.emeter_realtime energy_values[device.alias] = emeter_realtime["power"] From 237ca8f3030487c358d2a77e3dcc070f120519a1 Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 14:26:14 -0700 Subject: [PATCH 03/17] refactor so tests are dynamically udpated if db schema changes correct emissions calculations --- .gitignore | 3 ++ docker-compose.yml | 2 + modules/database.py | 25 ++++++--- modules/kasa_monitor.py | 31 +++++++++-- sql/01_init.sql | 14 +++++ sql/02_create_view_user.sh | 7 +++ sql/init.sql | 7 --- tests/conftest.py | 32 ++++++++++++ tests/test_database.py | 52 +++++++------------ tests/test_kasa_monitor.py | 9 ++-- visualization/power_and_co2_consumption.pbit | Bin 0 -> 7183 bytes 11 files changed, 127 insertions(+), 55 deletions(-) create mode 100644 sql/01_init.sql create mode 100644 sql/02_create_view_user.sh delete mode 100644 sql/init.sql create mode 100644 tests/conftest.py create mode 100644 visualization/power_and_co2_consumption.pbit diff --git a/.gitignore b/.gitignore index e2360c8..dbfac3f 100644 --- a/.gitignore +++ b/.gitignore @@ -159,5 +159,8 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +#powerbi files +*.pbix + #project specific ignores em_cache.json \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1d56aa7..ed35d9b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,8 @@ services: POSTGRES_DB: ${DB_NAME} POSTGRES_PASSWORD: ${DB_PASSWORD} POSTGRES_USER: ${DB_USER} + VIEW_USER: ${DB_VIEW_USER} + VIEW_USER_PASSWORD: ${DB_VIEW_USER_PASSWORD} volumes: - ./sql:/docker-entrypoint-initdb.d ports: diff --git a/modules/database.py b/modules/database.py index 25c2738..67b6e8f 100644 --- a/modules/database.py +++ b/modules/database.py @@ -9,17 +9,16 @@ def __init__(self, db_config): async def write_usage(self, energy_usage): self.conn = await asyncpg.connect(**self.db_config) try: - await self.conn.execute( - "INSERT INTO energy_usage (device, timestamp, power, avg_mg_co2) VALUES ($1, $2, $3, $4)", - energy_usage["device"], energy_usage["timestamp"], energy_usage["power"], energy_usage["avg_mg_co2"] - ) + sql_query = self._generate_insert_sql_query(energy_usage) + await self.conn.execute(sql_query, *energy_usage.values()) finally: await self.close() - async def read_usage(self): + async def read_usage(self, columns="*"): self.conn = await asyncpg.connect(**self.db_config) try: - result = await self.conn.fetch("SELECT * FROM energy_usage") + sql_query = self._generate_select_sql_query(columns) + result = await self.conn.fetch(sql_query) finally: await self.close() return result @@ -27,4 +26,16 @@ async def read_usage(self): async def close(self): if self.conn is not None and not self.conn.is_closed(): await self.conn.close() - self.conn = None \ No newline at end of file + self.conn = None + + def _generate_insert_sql_query(self, energy_usage): + columns = ', '.join(energy_usage.keys()) + values = ', '.join(['$' + str(i) for i in range(1, len(energy_usage) + 1)]) + return f"INSERT INTO energy_usage ({columns}) VALUES ({values})" + + def _generate_select_sql_query(self, columns="*"): + if columns == "*": + columns_str = "*" + else: + columns_str = ', '.join(columns) + return f"SELECT {columns_str} FROM energy_usage" \ No newline at end of file diff --git a/modules/kasa_monitor.py b/modules/kasa_monitor.py index a65d6f2..ee18b49 100644 --- a/modules/kasa_monitor.py +++ b/modules/kasa_monitor.py @@ -12,7 +12,7 @@ def __init__(self, local_lat=None, local_lon=None, local_grid_id=None, co2_api_p self.lon = None self.grid_id = None if (local_lat is None or local_lon is None) and local_grid_id is None: - #TODO use a geoip service to get the lat/lon + #TODO get the location from the smart plug; it should have it self.lat = os.getenv("LOCAL_LAT") self.lon = os.getenv("LOCAL_LON") elif local_grid_id is not None: @@ -57,6 +57,25 @@ async def _get_co2_data(self): return await self.co2_api.get_co2_by_latlon(self.lat, self.lon) async def monitor_energy_use_continuously(self, db, delay, timeout=None): + ''' + Monitor energy use continuously and store the data in the database. + + DB Schema: + device VARCHAR(255) NOT NULL, + timestamp TIMESTAMPTZ NOT NULL, + power_draw_watts real NOT NULL, + avg_emitted_mgco2e real, + grid_carbon_intensity_gco2perkwhr real, + PRIMARY KEY (device, timestamp) + + Args: + db (Database): The database to store the energy use data in. + delay (int): The number of seconds to wait between each update. + timeout (int, optional): The number of seconds to run the monitor for. Defaults to None, which will run forever. + + Returns: + None + ''' start_time = time.time() try: @@ -66,8 +85,14 @@ async def monitor_energy_use_continuously(self, db, delay, timeout=None): #Get average CO2 from API co2 = await self._get_co2_data() for device, power in energy_values.items(): - - energy_usage = {"device": device, "timestamp": datetime.now(timezone.utc), "power": power, "avg_mg_co2": co2} + #convert gird co2 to device actual co2 by taking the timespan (assuming constant power draw over that period) and + #multiplying by the grid co2 while also converting to mgCO2e + power_kwatts = power / 1000.0 #convert to kW from watts + hours = delay / 3600.0 #time in use in hours + co2emitted = hours * power_kwatts * co2 / 1000.0 #convert to mgCO2e + energy_usage = {"device": device, "timestamp": datetime.now(timezone.utc), + "power_draw_watts": power, "avg_emitted_mgco2e": co2emitted, + "grid_carbon_intensity_gco2perkwhr": co2} await db.write_usage(energy_usage) if timeout is not None and time.time() - start_time >= timeout: diff --git a/sql/01_init.sql b/sql/01_init.sql new file mode 100644 index 0000000..7cd9ce0 --- /dev/null +++ b/sql/01_init.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS energy_usage ( + device VARCHAR(255) NOT NULL, + timestamp TIMESTAMPTZ NOT NULL, + power_draw_watts real NOT NULL, + avg_emitted_mgco2e real, + grid_carbon_intensity_gco2perkwhr real, + PRIMARY KEY (device, timestamp) +); + +CREATE OR REPLACE VIEW energy_usage_view AS +SELECT * +FROM energy_usage +ORDER BY timestamp DESC +LIMIT 100000; diff --git a/sql/02_create_view_user.sh b/sql/02_create_view_user.sh new file mode 100644 index 0000000..02acfa3 --- /dev/null +++ b/sql/02_create_view_user.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE ROLE "$VIEW_USER" WITH LOGIN PASSWORD '$VIEW_USER_PASSWORD'; + GRANT SELECT ON energy_usage_view TO "$VIEW_USER"; +EOSQL diff --git a/sql/init.sql b/sql/init.sql deleted file mode 100644 index 28c0daa..0000000 --- a/sql/init.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE TABLE energy_usage ( - device VARCHAR(255) NOT NULL, - timestamp TIMESTAMPTZ NOT NULL, - power real NOT NULL, - avg_mg_co2 real, - PRIMARY KEY (device, timestamp) -); \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..157cdd8 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,32 @@ +import pytest +import re +from datetime import datetime, timezone +import pytz + +@pytest.fixture +def energy_usage(): + # Read the SQL file + with open('sql/01_init.sql', 'r') as file: + sql = file.read() + + # Parse the CREATE TABLE statement to get the column names and types + match = re.search(r'CREATE TABLE IF NOT EXISTS energy_usage \((.*)\)\s*;', sql, re.DOTALL) + column_definitions = re.split(r',(?![^(]*\))', match.group(1)) + columns = [re.search(r'^\s*(\w+)\s+(.+)', column_definition).groups() for column_definition in column_definitions] + + # Create the energy_usage dictionary with test values based on the column names and types + energy_usage = {} + for column_name, column_type in columns: + if 'VARCHAR' in column_type: + energy_usage[column_name] = 'test_value' + elif 'TIMESTAMPTZ' in column_type: + timestamp_str = "2022-01-01 00:00:00" + timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S") + timestamp = pytz.utc.localize(timestamp) # Make timestamp timezone aware and in UTC timezone + energy_usage[column_name] = timestamp + elif 'integer' in column_type: + energy_usage[column_name] = 100 + elif 'real' in column_type: + energy_usage[column_name] = 50.0 + + return energy_usage \ No newline at end of file diff --git a/tests/test_database.py b/tests/test_database.py index 201301f..f8306cb 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -5,8 +5,7 @@ from unittest.mock import patch, AsyncMock, Mock from modules.database import Database import config -from datetime import datetime, timezone -import pytz + @pytest.fixture(autouse=True) @pytest.mark.real_database @@ -26,11 +25,10 @@ async def setup_method(): print("Tearing down") @pytest.mark.asyncio -async def test_write_usage(): +async def test_write_usage(energy_usage): # Arrange db_config = {"user": "test", "password": "test", "database": "test", "host": "localhost"} db = Database(db_config) - energy_usage = {"device": "test_device", "timestamp": "2022-01-01 00:00:00", "power": 100, "avg_mg_co2": 50} with patch('asyncpg.connect', new_callable=AsyncMock) as mock_connect: mock_connect.return_value.execute = AsyncMock() @@ -42,9 +40,10 @@ async def test_write_usage(): # Assert mock_connect.assert_called_once_with(**db_config) + expected_query = db._generate_insert_sql_query(energy_usage) mock_connect.return_value.execute.assert_called_once_with( - "INSERT INTO energy_usage (device, timestamp, power, avg_mg_co2) VALUES ($1, $2, $3, $4)", - energy_usage["device"], energy_usage["timestamp"], energy_usage["power"], energy_usage["avg_mg_co2"] + expected_query, + *energy_usage.values() ) mock_connect.return_value.close.assert_called_once() @@ -63,19 +62,16 @@ async def test_read_usage(): # Assert mock_connect.assert_called_once_with(**db_config) - mock_connect.return_value.fetch.assert_called_once_with("SELECT * FROM energy_usage") - mock_connect.return_value.close.assert_called_once() + expected_query = db._generate_select_sql_query() + mock_connect.return_value.fetch.assert_called_once_with(expected_query) + mock_connect.return_value.close.assert_called_once() @pytest.mark.asyncio @pytest.mark.real_database -async def test_write_usage_realdb(): +async def test_write_usage_realdb(energy_usage): # Arrange db_config = config.db_config db = Database(db_config) - timestamp_str = "2022-01-01 00:00:00" - timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S") - timestamp = pytz.utc.localize(timestamp) # Make timestamp timezone aware and in UTC timezone - energy_usage = {"device": "test_device", "timestamp": timestamp, "power": 100.0, "avg_mg_co2": 50.0} # Act await db.write_usage(energy_usage) @@ -83,33 +79,27 @@ async def test_write_usage_realdb(): # Assert conn = await asyncpg.connect(**db_config) try: + expected_query = db._generate_insert_sql_query(energy_usage) result = await conn.fetchrow("SELECT * FROM energy_usage WHERE device = $1", energy_usage["device"]) assert result is not None - assert result["device"] == energy_usage["device"] - assert result["timestamp"] == energy_usage["timestamp"] - assert result["power"] == energy_usage["power"] - assert result["avg_mg_co2"] == energy_usage["avg_mg_co2"] + for key, value in energy_usage.items(): + assert result[key] == value + finally: await conn.execute("DELETE FROM energy_usage") await conn.close() @pytest.mark.asyncio @pytest.mark.real_database -async def test_read_usage_realdb(): +async def test_read_usage_realdb(energy_usage): # Arrange db_config = config.db_config db = Database(db_config) - timestamp_str = "2022-01-01 00:00:00" - timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S") - timestamp = pytz.utc.localize(timestamp) # Make timestamp timezone aware and in UTC timezone - energy_usage = {"device": "test_device", "timestamp": timestamp, "power": 100.0, "avg_mg_co2": 50.0} - + conn = await asyncpg.connect(**db_config) try: - await conn.execute( - "INSERT INTO energy_usage (device, timestamp, power, avg_mg_co2) VALUES ($1, $2, $3, $4)", - energy_usage["device"], energy_usage["timestamp"], energy_usage["power"], energy_usage["avg_mg_co2"] - ) + insert_query = db._generate_insert_sql_query(energy_usage) + await conn.execute(insert_query, *energy_usage.values()) finally: await conn.close() @@ -118,12 +108,8 @@ async def test_read_usage_realdb(): # Assert assert len(result) > 0 - assert result[0]["device"] == energy_usage["device"] - # Convert the original timestamp to the client's timezone - timestamp_client_tz = energy_usage["timestamp"].astimezone(timezone.utc) - assert result[0]["timestamp"] == timestamp_client_tz - assert result[0]["power"] == energy_usage["power"] - assert result[0]["avg_mg_co2"] == energy_usage["avg_mg_co2"] + for key, value in energy_usage.items(): + assert result[0][key] == value conn = await asyncpg.connect(**db_config) try: diff --git a/tests/test_kasa_monitor.py b/tests/test_kasa_monitor.py index 3a27d96..3ba73c1 100644 --- a/tests/test_kasa_monitor.py +++ b/tests/test_kasa_monitor.py @@ -65,7 +65,7 @@ async def test_monitor_energy_use_real_device(): @pytest.mark.asyncio @pytest.mark.real_device @pytest.mark.real_database -async def test_monitor_energy_use_continuously_integration(): +async def test_monitor_energy_use_continuously_integration(energy_usage): # Arrange db_config = config.db_config db = Database(db_config) @@ -82,10 +82,9 @@ async def test_monitor_energy_use_continuously_integration(): result = await db.read_usage() assert len(result) > 0 - assert result[0]["device"] is not None - assert result[0]["timestamp"] is not None - assert result[0]["power"] is not None - assert result[0]["avg_mg_co2"] is not None + for key, value in energy_usage.items(): + assert result[0][key] is not None + #empty the db again conn = await asyncpg.connect(**db_config) diff --git a/visualization/power_and_co2_consumption.pbit b/visualization/power_and_co2_consumption.pbit new file mode 100644 index 0000000000000000000000000000000000000000..fc6bd13937b45fd15725b1175b9942da58394069 GIT binary patch literal 7183 zcmbVx2UJr{w|0<@qN374EPx8qq)3q_(h;QBfT1UJNFaa&kSbMr3!!&N1O(|tA)y!P z9YT}dq)NHG-@X6$zk2Wg-nITYYtCAGoq6`mo;fr7nZ4gA6B6A4003lwertp-v$?Sm z#lIu~fE2(4(6MlK0Xx{=%cKK5PXDXfe=q#zeP+C-pWXrhZ+=td%%-Zfb=jQv3IK5X zQk9{MgFV#39%`cH?r7lx9&mS_42`#g}XcR1kALhJ|@7ROWt1a`az zN<{ibw`nABR9juw1t@N`n;QI*;?^}_;c~uiMoKWm=t{IK>qO8Y#*X8ZIQR&w&Y=lH zwQk(cTQ0@4i#@TooO9A)%kims*M`mPXT8g}D?W}4N*G(yIHsX@GuE6@aiujQsSQ53 zE-$&Q@#Dk$$ZHpV45mFeRmq=pFCFeU8&fasoi=X;t1q*uEi!7z2)r<6%=6yibK*T~ zJSX{m?kvJzn^Amhvd@g}&%s+0gVtrC9(Al7%L`gvL)zz7-4Fb7g z7y)y0z>jMAneuD!q#|&=ro%SUw6)-bEKKHWm(-9-YD^yJN?P~^pC4yhZF-IqL7hoI zzcyyDz{5^>GhKI>1}*6fwS-432E{sqDFAw8%Em3% z!T=dGEd6NqW;PR+J)St90aKXR?sM_cznh-y1L4vK7Ry3G=3r9;j?SDI9=1ptTF2#G zYX!p&wrt_jPhI&j4|1f*pH@~P^(IT?V!}7xA3XeY*G)`unHkH}bJlWreHb?Qyu#2l z)xw}JH)tKCyC}Le{e13t)wUh+C7<{@_f8_EQ6`m_lpx=lOA*}N=}YZcMo*Hm1J7DvAXra7r>$yhauStA>J#mm;` z;y);)R30kOP?7Faj=D3}+l`j!oub+u89uxC?$8sobSigRy|8H-P_A5XK{vKcOc+Uw zx%;0jm+lFt7~mRk?B!$rcDfn)$&1<<&Pm;h4TGfYBCw;b+|^Uaz|uH-hmuiTVl8d} zM`c^A`1k`F*Ht!LAtHWa?xg(`4X@WeuIF8jnkg3EBdvIB6xvG)3%BxNHVk{S9P1D{ zGlFDJ{d)J}Q3HwJJuva^sbhP4CK+L0)_6N2uBai~3qdpej^5#! ze3-DO8c$}?p%c4yYD}lQxHtXTm?v@_h)fvtUGPrZzhGIo$O{Gq<`%+w$x8MT|x`qo&U|=w79! zk3F^JXJlxSM190Jv=45nK#gQ;W1^LpPSfap_iP`PxXCi;bl1iXmA5V7yYS4Y>F>!Z z@g=~V2dYW73b=&N{$&fs@mbvH;G%pAFUSE{VVq9CN>e!ww5N5g!) zRj6V|M~9|v5cRMg{WMPTQ~MTS*m|PGv3ylZdvoS>Ig^?T^)uq%kK`!MA6+m403ewV z0HFNqNCtzfoI!RfAa@5C^q&soxEMp_Z&Eh{mh)mI48g?*_!PKw=?#fs35BRp+myG- zL^0DV-iH(IB$*`H{uWlf$m-Oo^&BovzFoHJSJ6sekE0jc3((vLbD8&QA+blvJ5r-~*m|Eh22->daVrN8urRNbDOPTB|@r}lE;H=!6%6?&A zC$wuww9!Ukh*9}hTry{N7DNl0jhlJPTp6^+u%abS#a2_CLt)5HDemmNR9ADNC8tfF z$b6Aj1|=3%A3;D0h;n7rY^!+}^dD?VK3v@GK-Yck5;M5Vpl!Dmy5k?{uCIgMq!_oWNEvzTM_(tw=pCz&Pp#Ic=jo$E_{i0-^p_k7e;CiceWqJbH5|m7 zNPHca8<*bq4v3vLgSPXMoVxmftg7-5h`X=bz27xb80+x4QZpI0ZrwW9J#zbiF_4=f z^54RrbMAdp!W70g!;&3{wd!QkIMCE}hFC^O4j=1?khOTLFhd1V6(?OqSOaI(h_`lN zWANI8>47Ry&4Go|WJZ}dGJ4k2cyuS+z@i&XSWA)(Y=u#qsqvbmmg*B3lkY{N4IX5V z(8FVex!L6KAF@P1;o|a`${2gor;k8rcC$S}P*0$XcO*(YD5lgM`N~L@eb(ixs}fBa zjzP}AjIZn<>Qlf+)r!%G>j6EA#tOhwbK4o^0<+J~-;NSg+OmR{T}I*70nVG02MkYm z?b$XN6Tjeu+lN<(Qp5YoysdULc`apJ=n-gg^8&xSvvn#wJ= zYoW0giFQp5EzYafwv*TUoO2O11QCLhU$&RRFy>FLu3Dk?DD);-jLQ5mlbtWcEQ5^$ z=`U=?F6MYYM!yh1DUL^1M7X`c>9;gbAzPJ`Y)gA>_yQ3=TXZTUJ>oVlKwaYZ*_Y+#NA?ySi;r5fK}-OL1#d7sCf(jJ{xgHtvRk zkqDTtKlgw)eyA&u-n8n>P-tq^cTn)oBH}5ibc~1>2 z;f77pWR`6=m&%1Gb$_v>g|?|6{3_1Fz}5?&eKYN*;`wwWQbx^RKrHh@^2x2D>rhMg zB3-nRmv2R-0j32uh2xyXdjoBYBC_0Tb>5S_zxr+!c?CLRYdhvdFHr9MZ^;BW;!~D@ zI}LkQ(=mB;R=%a%iK9pJ4{H zH)l(Z1aqL{1fuepgpDL;Mfj;qSe$J_M+@+92emQ*uC%bplRm8wnhp1~@E;)uJ7=m= z#?BVCCws)TE5^4Ltw?N4y%7_s3K4`mpkAtz;+ScE>qsrDNdXmir4{hO5qMKstwSqsvsIU2o%Ri3v#OrN0JS-1Yh(U*D=Pc~el*kWXm3KXU zT4aW$6lT0%EcPRVKZtiaog-hH6n7H5yhMxrKrgp4#@fFHGk|$JIhF>FO>WIWAp{1n zdezsu?lsfe*@8(+G!`OjV@crl}r&XH~5#4sfl&$B=dx359id(tYaZb4WzM=$u|iu`HzGZaEWN z6ZqPKzu*Sk{&8b3n-k^QC)36h=Muv)3R(=^q6FKigvB#8=M@8}1&+aUoMwsKje4u& zTc7*Qxp_60T?sjaH^@Rd`j}gXb1mkA^mnc`B~25Xb?L5EFZmYPCD+olfI`9cRxbZU zvjRSLQhepTDck!6%7xG3bPM4&I%%q}l=V|r9^lUYIa5!cpYq^VwBjbRIw_lN?v=D` z(+<91?|p0(-q_JFlbXos0^0md~?eOxR|4&AGg;!JfN!__LQiTiuK0KA^ z)aMSk<)x!k^D8G)wsX(TE-moFPJHc->NTFHVi8Gkk&drrGp8K4sSiIw=%I#ksAOfS z{;fo)-MGY9f6%&jF13L>Y0wBwhD=@c4ptK7mdvOBu|QeFIc^UB$D#ydsbZj)F$oqL z06_9`y#Ga_G@&3U*i6I1#R2AQX5qr4X$^9=F#o^uOOT6&)@9J*@{^4IGk#5NK0a=U z%Rh&dhV&-vCxmaF#Ys)&$d&wn#6GyL@1A*O5<-Xlr}puYwnr<_LZS68>ys z=5~vY86>Q9rIX6_AROl(_F!KAO6(%@wL0~;ZbAO9;=j3hjf^5A9@efAXO*VtF*oz> zK3$|*b9i>3Y5H`ZFq#D0Da^_$FxccKKC>REe$#y(VT31(rLSkDSFvqTWU8`XeJ27# z=a{;%O%(bNy{ATPQ6i@nZxj!OgwS_dXup9);$GfT@mm-z<;)e>Qf>81e05tmuK!*2 zP*if-^sY!6&*w45Y~$5~1JUNHl(1-C=}5tDqR?HWcA$+Px;j>_6?+Wntn*cu>!v4U zzoYYA%(&7*?WU4tAx>u(;${0R)!;^<4%ymw-hC>|&|k?r^N(jD+RhpmufG8^KeZ&q%rOE(+Y@iil_1-)`KJBiK zD?G}!-avwf%BX$l-gk7-u+~B#g?e#>MC6;&yet#wfWau!pBZOJ$S0jCdAB+~tK@*X z)<{l98vz2JJh-nxSpTf*eL>dABi#&t3Pdj-_r3Y#5Uie;jBF=Dk=uF3Q+(KtSM+AO zm;$pLgjs)KP>Fl&k@1fJ7ojG?iP^hk(_Hh@ODlvq%56(xQ73)}c3(~&nBi9~2klZf5JybEv{+hI=GJJ!IQY>+* z!VXrW?kH2;WW2x5(J-q-*!L1cqqXun+}tj{WWWY&=;sZhir3;ws>vQg0;Qi{q5M4b zeHPrqVQIj&{bYEs401#{Vaf8~F*hqN!qewkvyW$JNFz0uFtl&a-ra`pE5>0na!`m( zpyjZnuZ{%Mvr-R^5Elk+$kP-pF$uu9cPX@lhkb5Eouxep(D*T;Q+i06*(9pcCpXFF zpFw>5(8aTb)aIxC+2U=(w`)cW7lVeDTdf4?R?T}Bt;yKfC*yDS>J>KGK)^>8{+&Uc z6fde?Ien-fR~fH~AYuwMa!Vq==I|(2EEUxh7mzUbp-!S-LngEM;H6FpLi$XwG3ulv zR_DT5YG|M~Il1uGXM4R#2xb0w0O$j}CiY7u3-t83Mg;mCo5y{EQtYzDu^mm?8u)f5 zJ1kfOOm~{%=A+Qg#JwAKzp@|rfX9^#x8F-cM~TyYpRA4bgkRB=1JCOZWX=vTeI&LH zkS+kjvBV>Xh2*Q6MD z1ZQLCnSDX`4x|jqm2R_n<~=cdK4NL_L)CcQGcpq&H+ntqM`#a0IHm5-@2-iy)|iLw z&tiD&wHiC?suV-4l1|(0-HsZ@>r49rQwcUlCvt0b?QYn>B@FwL#Xsaos`Ahmb=w~G zZW1*jel%F=Ad*f{PG*3Y>V z=6DDdB*iMf6MWHVeoCv(z7=uhVuT;K#qM!t@LYZ`XeMh)a%(-yqZf0Yzjlj%4Xran z@Vj%Gj%=j0y7a!5m-XhqJg^ze84PuQ3AQ)?-#oFI`P;Wuef)`ltlnrVS8X?24V~VA-^;Cb zLiEP>!a6w9NR!1FSRU7*$TWIK0R0pijRw%gy4~^bb`Yk6aacVmPsoh33je8-s4Y&@ zWM=SnZ4x_Rh-{dx9?$PRJTkc4fK%K;D4rlAo{(%zbnw&m&XvbPfaYrfkP=+PqA-8UXoFJ zunu?qYOd>2&(Dg&smo@c0=+rjmH4M_9oD9JnYrb04V@T58$gN^&UuUzX4Dc@_LQUB za8g6~TFvay@|Pv0md+Zumax2Xq4*i(+GVzd;0hPu|CPGABo)7v|4QWi3Hm)r^LK>u zrSLzbZ2na6d#2*=3W6?woj>vxe}aFHsr?Du2>HdKIrlea%nzmulFqZn`fdzk;f6zb2;e&4iz?~IJ@ bFT;M{!f%v`N&ad>e)&jU=IE;F|C0U#7{~9y literal 0 HcmV?d00001 From 79d417d02275e1dd0cf4a7344eeda4a4619d926b Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 16:43:04 -0700 Subject: [PATCH 04/17] Add test action --- .github/workflows/test.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..f19d8d3 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,34 @@ +# .github/workflows/test.yml +name: Run tests + +on: + pull_request: + branches: + - main + - ppe + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.12 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Set up Docker Compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose + docker-compose up -d + + - name: Run tests + run: pytest -m "not real_device and not real_em_api" \ No newline at end of file From 9e9c3507a17015019fc85d0ec9095289b7178a4d Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 16:50:25 -0700 Subject: [PATCH 05/17] lookup github secrets for deploy --- .github/workflows/test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f19d8d3..8030060 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,6 +25,14 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Set up Docker Compose + env: + DB_HOST: ${{ secrets.DB_HOST }} + DB_USER: ${{ secrets.DB_USER }} + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} + DB_NAME: ${{ secrets.DB_NAME }} + DB_PORT: ${{ secrets.DB_PORT }} + DB_VIEW_USER: ${{ secrets.DB_VIEW_USER }} + DB_VIEW_PASSWORD: ${{ secrets.DB_VIEW_PASSWORD }} run: | sudo apt-get update sudo apt-get install -y docker-compose From 088c23af93acb43c49a1c7364673e3afad397d20 Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 16:53:42 -0700 Subject: [PATCH 06/17] Define pythonpath in action --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8030060..48d139d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,4 +39,6 @@ jobs: docker-compose up -d - name: Run tests + env: + PYTHONPATH: ${{ github.workspace }} run: pytest -m "not real_device and not real_em_api" \ No newline at end of file From ba55ba19bce52d22f1d354fdcd5ce5868c5a6dc2 Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 17:12:01 -0700 Subject: [PATCH 07/17] Fix db view user password secret name --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 48d139d..fdb1858 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,7 +32,7 @@ jobs: DB_NAME: ${{ secrets.DB_NAME }} DB_PORT: ${{ secrets.DB_PORT }} DB_VIEW_USER: ${{ secrets.DB_VIEW_USER }} - DB_VIEW_PASSWORD: ${{ secrets.DB_VIEW_PASSWORD }} + DB_VIEW_USER_PASSWORD: ${{ secrets.DB_VIEW_USER_PASSWORD }} run: | sudo apt-get update sudo apt-get install -y docker-compose From acd97d56836cfc41c4285dc6fa4d051c9e792c71 Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 17:33:28 -0700 Subject: [PATCH 08/17] Fix env variable lookup for action --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fdb1858..b591ccb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,4 +41,5 @@ jobs: - name: Run tests env: PYTHONPATH: ${{ github.workspace }} + EM_CACHE_EXPIRY_MINS: ${{ env.EM_CACHE_EXPIRY_MINS }} run: pytest -m "not real_device and not real_em_api" \ No newline at end of file From dbbd039f3e41093dede6e7138539ba5ad332ec5b Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 17:37:33 -0700 Subject: [PATCH 09/17] update cache expiry for action --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b591ccb..d2e12cf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,5 +41,5 @@ jobs: - name: Run tests env: PYTHONPATH: ${{ github.workspace }} - EM_CACHE_EXPIRY_MINS: ${{ env.EM_CACHE_EXPIRY_MINS }} + EM_CACHE_EXPIRY_MINS: 30 run: pytest -m "not real_device and not real_em_api" \ No newline at end of file From 73f91a6dd5f0dbf720503f21a672b95a396311e3 Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 17:50:00 -0700 Subject: [PATCH 10/17] refactor env vars to be at job level --- .github/workflows/test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d2e12cf..05e7adc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,6 +10,14 @@ on: jobs: test: runs-on: ubuntu-latest + env: + DB_HOST: ${{ secrets.DB_HOST }} + DB_USER: ${{ secrets.DB_USER }} + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} + DB_NAME: ${{ secrets.DB_NAME }} + DB_PORT: ${{ secrets.DB_PORT }} + DB_VIEW_USER: ${{ secrets.DB_VIEW_USER }} + DB_VIEW_USER_PASSWORD: ${{ secrets.DB_VIEW_USER_PASSWORD }} steps: - name: Checkout code uses: actions/checkout@v3 @@ -25,14 +33,6 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Set up Docker Compose - env: - DB_HOST: ${{ secrets.DB_HOST }} - DB_USER: ${{ secrets.DB_USER }} - DB_PASSWORD: ${{ secrets.DB_PASSWORD }} - DB_NAME: ${{ secrets.DB_NAME }} - DB_PORT: ${{ secrets.DB_PORT }} - DB_VIEW_USER: ${{ secrets.DB_VIEW_USER }} - DB_VIEW_USER_PASSWORD: ${{ secrets.DB_VIEW_USER_PASSWORD }} run: | sudo apt-get update sudo apt-get install -y docker-compose From 3e5ade47abd2aeaa444d8206cc827cd7d3ed59d5 Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 18:14:51 -0700 Subject: [PATCH 11/17] additional test debugging --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 05e7adc..117cc7e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,6 +37,8 @@ jobs: sudo apt-get update sudo apt-get install -y docker-compose docker-compose up -d + #output the postgres db host name + docker ps - name: Run tests env: From 64e59d6a0987c8e7522f238d07de53e4d3ed93f9 Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 18:23:55 -0700 Subject: [PATCH 12/17] debug failing test --- tests/test_database.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/test_database.py b/tests/test_database.py index f8306cb..2650973 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -66,28 +66,27 @@ async def test_read_usage(): mock_connect.return_value.fetch.assert_called_once_with(expected_query) mock_connect.return_value.close.assert_called_once() -@pytest.mark.asyncio -@pytest.mark.real_database -async def test_write_usage_realdb(energy_usage): - # Arrange - db_config = config.db_config - db = Database(db_config) +# @pytest.mark.asyncio +# @pytest.mark.real_database +# async def test_write_usage_realdb(energy_usage): +# # Arrange +# db_config = config.db_config +# db = Database(db_config) - # Act - await db.write_usage(energy_usage) +# # Act +# await db.write_usage(energy_usage) - # Assert - conn = await asyncpg.connect(**db_config) - try: - expected_query = db._generate_insert_sql_query(energy_usage) - result = await conn.fetchrow("SELECT * FROM energy_usage WHERE device = $1", energy_usage["device"]) - assert result is not None - for key, value in energy_usage.items(): - assert result[key] == value +# # Assert +# conn = await asyncpg.connect(**db_config) +# try: +# result = await conn.fetchrow("SELECT * FROM energy_usage WHERE device = $1", energy_usage["device"]) +# assert result is not None +# for key, value in energy_usage.items(): +# assert result[key] == value - finally: - await conn.execute("DELETE FROM energy_usage") - await conn.close() +# finally: +# await conn.execute("DELETE FROM energy_usage") +# await conn.close() @pytest.mark.asyncio @pytest.mark.real_database From 0bc1ce77246094c4ff33cbb6bb86abd018ea5a0b Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 18:33:06 -0700 Subject: [PATCH 13/17] more debugging for docker db --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 117cc7e..20e016c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,8 @@ jobs: docker-compose up -d #output the postgres db host name docker ps - + docker exec -it kasa_carbon-db-1 psql -U postgres -c "\l" + - name: Run tests env: PYTHONPATH: ${{ github.workspace }} From 4a6fa1b41fe2d3ac16fdb8dd32b3e747abae6fe5 Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 18:35:24 -0700 Subject: [PATCH 14/17] fix typo --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 20e016c..00941bb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: docker-compose up -d #output the postgres db host name docker ps - docker exec -it kasa_carbon-db-1 psql -U postgres -c "\l" + docker exec -it kasa_carbon_db_1 psql -U postgres -c "\l" - name: Run tests env: From 8ada3f6849fb4075eeaac31ed0705779c89dc5e0 Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 18:39:43 -0700 Subject: [PATCH 15/17] fix docker command --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 00941bb..f5ed38f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: docker-compose up -d #output the postgres db host name docker ps - docker exec -it kasa_carbon_db_1 psql -U postgres -c "\l" + docker exec kasa_carbon_db_1 psql -U postgres -c "\l" - name: Run tests env: From 075c1c90be658e23f43e2c4fafa27e6eb46d3734 Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 18:44:12 -0700 Subject: [PATCH 16/17] sleep in action --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f5ed38f..4b5f121 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,8 @@ jobs: docker-compose up -d #output the postgres db host name docker ps - docker exec kasa_carbon_db_1 psql -U postgres -c "\l" + sleep 10 + docker exec kasa_carbon_db_1 psql -U ${{ secrets.DB_USER }} -c "\l" - name: Run tests env: From d903de6591079f580e683c1dd9afb2d3894fa6fe Mon Sep 17 00:00:00 2001 From: scottcha Date: Thu, 30 Nov 2023 18:46:21 -0700 Subject: [PATCH 17/17] reenable db test --- tests/test_database.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/test_database.py b/tests/test_database.py index 2650973..1336a8a 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -66,27 +66,27 @@ async def test_read_usage(): mock_connect.return_value.fetch.assert_called_once_with(expected_query) mock_connect.return_value.close.assert_called_once() -# @pytest.mark.asyncio -# @pytest.mark.real_database -# async def test_write_usage_realdb(energy_usage): -# # Arrange -# db_config = config.db_config -# db = Database(db_config) +@pytest.mark.asyncio +@pytest.mark.real_database +async def test_write_usage_realdb(energy_usage): + # Arrange + db_config = config.db_config + db = Database(db_config) -# # Act -# await db.write_usage(energy_usage) + # Act + await db.write_usage(energy_usage) -# # Assert -# conn = await asyncpg.connect(**db_config) -# try: -# result = await conn.fetchrow("SELECT * FROM energy_usage WHERE device = $1", energy_usage["device"]) -# assert result is not None -# for key, value in energy_usage.items(): -# assert result[key] == value + # Assert + conn = await asyncpg.connect(**db_config) + try: + result = await conn.fetchrow("SELECT * FROM energy_usage WHERE device = $1", energy_usage["device"]) + assert result is not None + for key, value in energy_usage.items(): + assert result[key] == value -# finally: -# await conn.execute("DELETE FROM energy_usage") -# await conn.close() + finally: + await conn.execute("DELETE FROM energy_usage") + await conn.close() @pytest.mark.asyncio @pytest.mark.real_database