From eaf1e193acee67c1e25bfc36661a031b3870730d Mon Sep 17 00:00:00 2001 From: Erik van Velzen Date: Mon, 8 Apr 2024 17:28:46 +0200 Subject: [PATCH] Add PyCharm debugging in development --- docker/config/python.example.env | 4 ++- src/Dockerfile | 2 +- src/holon/models/config/query_and_convert.py | 4 +-- src/holon/services/query_and_convert.py | 26 ++++++++++++++++---- src/manage.py | 21 +++++++++++----- src/pipit/settings/__init__.py | 8 ++++++ src/pipit/settings/base.py | 6 ++++- src/requirements/base.txt | 1 + src/requirements/local.txt | 1 + src/requirements/prod.txt | 1 - 10 files changed, 57 insertions(+), 17 deletions(-) diff --git a/docker/config/python.example.env b/docker/config/python.example.env index b9fa05f77..0ae7542b0 100644 --- a/docker/config/python.example.env +++ b/docker/config/python.example.env @@ -11,4 +11,6 @@ SENTRY_ENVIRONMENT=development SECRET_KEY=asdfasdf ALLOWED_HOSTS=* DOMAIN_HOST="https://pizzaoven.holontool.nl" - +# Set to enable remote debugging with PyCharm +PYCHARM_IP=172.17.0.1 +PYCHARM_PORT=9222 diff --git a/src/Dockerfile b/src/Dockerfile index 9e02eca2a..93dc5a73d 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -21,7 +21,7 @@ WORKDIR /app ADD . /app/ RUN pip install --upgrade pip \ - && pip install -r requirements/prod.txt --no-cache-dir \ + && pip install -r requirements/local.txt --no-cache-dir \ && pip install ipython ipdb EXPOSE 8000 diff --git a/src/holon/models/config/query_and_convert.py b/src/holon/models/config/query_and_convert.py index 25b958190..58185970c 100644 --- a/src/holon/models/config/query_and_convert.py +++ b/src/holon/models/config/query_and_convert.py @@ -20,8 +20,8 @@ class QueryAndConvertConfig(ClusterableModel): name = models.CharField(max_length=255, null=False, blank=True) api_url = models.URLField( - default="https://beta-engine.energytransitionmodel.com/api/v3/scenarios/", - db_comment="This is currently not used and should be removed", + # or https://beta-engine.energytransitionmodel.com/api/v3/scenarios/ + default="https://engine.energytransitionmodel.com/api/v3/scenarios/", ) etm_scenario_id = models.IntegerField() diff --git a/src/holon/services/query_and_convert.py b/src/holon/services/query_and_convert.py index 23ab55cf3..52db0b346 100644 --- a/src/holon/services/query_and_convert.py +++ b/src/holon/services/query_and_convert.py @@ -26,12 +26,18 @@ qc_logger = HolonLogger("QConfig") -# Hardcoded because not bound to change at any point during this project -CONFIG_KPIS = { +# Configuration to map ETM outputs to Holon KPI's at regional and national level. +# Hardcoded because not bound to change at any point during this project. +UPSCALING_KPI_CONFIGS = { "api_url": "https://engine.energytransitionmodel.com/api/v3/scenarios/", "config": { + # Holon KPI "sustainability": { - "value": {"type": "query", "data": "value", "etm_key": "dashboard_renewability"}, + "value": { + "type": "query", + "data": "value", + "etm_key": "dashboard_renewability" + }, "convert_with": [ { "type": "static", @@ -43,6 +49,7 @@ } ], }, + # Holon KPI "self_sufficiency": { "value": { "type": "query", @@ -50,6 +57,7 @@ "etm_key": "kpi_self_sufficiency_local_production_of_primary_demand", } }, + # Holon KPI "netload": { "value": { "type": "query", @@ -57,7 +65,14 @@ "etm_key": "kpi_relative_future_load_mv_hv_transformer", } }, - "costs": {"value": {"type": "query", "data": "value", "etm_key": "total_costs"}}, + # Holon KPI + "costs": { + "value": { + "type": "query", + "data": "value", + "etm_key": "total_costs" + } + }, }, } @@ -76,6 +91,7 @@ def __init__( self.etm_scenario_id = config_db.etm_scenario_id self.anylogic_outcomes = anylogic_outcomes self.scenario_aggregate = scenario_aggregate + # used in cost-benefit to distribute costs over actors self.distribution_keys = {} @property @@ -201,7 +217,7 @@ def log_costs(etm_scenario_id: int, config, cost_components: dict): def upscaling(config: QConfig): new_scenario_id = etm_service.scale_copy_and_send(config.etm_scenario_id, config.queries) - kpis = etm_service.retrieve_results(new_scenario_id, copy(CONFIG_KPIS)) + kpis = etm_service.retrieve_results(new_scenario_id, copy(UPSCALING_KPI_CONFIGS)) ETMConnect.log_upscaling(config.etm_scenario_id, new_scenario_id, config, kpis) diff --git a/src/manage.py b/src/manage.py index add63a2ff..b250720ee 100755 --- a/src/manage.py +++ b/src/manage.py @@ -21,13 +21,13 @@ def if_exists_load_env(name: str) -> None: if os.path.exists(env_file): dotenv.load_dotenv(env_file, override=True) +def enable_pycharm_debugger(): + if settings.PYCHARM_IP and settings.PYCHARM_PORT: + import pydevd_pycharm + # we could do this on a per-request basis if convenient. + pydevd_pycharm.settrace(settings.PYCHARM_IP, port=settings.PYCHARM_PORT, stdoutToServer=True, stderrToServer=True) -def main(): - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pipit.settings.prod") - - if_exists_load_env(".env") - if_exists_load_env(".env.local") - +def enable_vscode_debugger(): # enable vs code remote debugging # https://github.com/Microsoft/PTVS/issues/1057 if settings.DEBUG and settings.VS_CODE_REMOTE_DEBUG and os.environ.get("RUN_MAIN"): @@ -35,6 +35,15 @@ def main(): ptvsd.enable_attach(address=("0.0.0.0", 5678)) +def main(): + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pipit.settings.prod") + + if_exists_load_env(".env") + if_exists_load_env(".env.local") + + enable_pycharm_debugger() + enable_vscode_debugger() + execute_from_command_line(sys.argv) diff --git a/src/pipit/settings/__init__.py b/src/pipit/settings/__init__.py index 66c5f540b..6b8aaf5a2 100644 --- a/src/pipit/settings/__init__.py +++ b/src/pipit/settings/__init__.py @@ -16,6 +16,14 @@ def get_env(name, default=None): raise ImproperlyConfigured(f"Set the {name} env variable") +def get_env_int(name: str, default: int = None): + string_value = os.getenv(name) + if string_value is None: + return default + + return int(string_value) + + def get_env_bool(name: str, default: bool = None) -> bool: if name in os.environ: string_value = os.environ[name].upper() diff --git a/src/pipit/settings/base.py b/src/pipit/settings/base.py index ffe53d6dc..83c3918e4 100644 --- a/src/pipit/settings/base.py +++ b/src/pipit/settings/base.py @@ -6,7 +6,7 @@ import os from typing import Optional -from pipit.settings import get_env, get_env_bool +from pipit.settings import get_env, get_env_bool, get_env_int # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(__file__)) @@ -17,6 +17,10 @@ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = get_env("SECRET_KEY") +# PyCharm or IntelliJ debugger +PYCHARM_IP = get_env("PYCHARM_IP") +PYCHARM_PORT = get_env_int("PYCHARM_PORT") + # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True diff --git a/src/requirements/base.txt b/src/requirements/base.txt index be81326f0..b4b17b726 100644 --- a/src/requirements/base.txt +++ b/src/requirements/base.txt @@ -1,6 +1,7 @@ Django==4.2.6 boto3>1.19,<1.25 django-storages==1.14.2 +django-storages[azure] psycopg==3.1.12 sentry_sdk==1.32.0 python-dotenv==0.21 diff --git a/src/requirements/local.txt b/src/requirements/local.txt index 61087c76e..fd78eb57d 100644 --- a/src/requirements/local.txt +++ b/src/requirements/local.txt @@ -2,6 +2,7 @@ -r test.txt # Add local extra requirements here (django-debug etc) +pydevd-pycharm~=241.14494.241 black django-debug-toolbar isort diff --git a/src/requirements/prod.txt b/src/requirements/prod.txt index 063734061..bf0c9be30 100644 --- a/src/requirements/prod.txt +++ b/src/requirements/prod.txt @@ -2,4 +2,3 @@ -r base.txt # Add production extra requirements here -django-storages[azure] \ No newline at end of file