Merge pull request #426 from openego/420-feature-implement-database-u…

…sing-test-oep-environment

420 feature implement database using test oep environment

.gitignore

@@ -24,3 +24,4 @@ eDisGo.egg-info/
 # exclude .json files in opf
 /edisgo/opf/opf_solutions/*.json
 /edisgo/opf/eDisGo_OPF.jl/.vscode
+.vscode/settings.json

edisgo/io/db.py

@@ -149,7 +149,7 @@ def ssh_tunnel(cred: dict) -> str:
     return str(server.local_bind_port)
 
 
-def engine(path: Path | str, ssh: bool = False) -> Engine:
+def engine(path: Path | str = None, ssh: bool = False) -> Engine:
     """
     Engine for local or remote database.
 
@@ -167,16 +167,15 @@ def engine(path: Path | str, ssh: bool = False) -> Engine:
         Database engine
 
     """
-    cred = credentials(path=path)
 
     if not ssh:
+        database_url = "toep.iks.cs.ovgu.de"
         return create_engine(
-            f"postgresql+psycopg2://{cred['POSTGRES_USER']}:"
-            f"{cred['POSTGRES_PASSWORD']}@{cred['HOST']}:"
-            f"{cred['PORT']}/{cred['POSTGRES_DB']}",
+            "postgresql+oedialect://:@" f"{database_url}",
             echo=False,
         )
 
+    cred = credentials(path=path)
     local_port = ssh_tunnel(cred)
 
     return create_engine(

edisgo/io/dsm_import.py

@@ -6,13 +6,13 @@
 
 import numpy as np
 import pandas as pd
-import saio
 
 from sqlalchemy.engine.base import Engine
 
 from edisgo.io.db import session_scope_egon_data
 from edisgo.io.timeseries_import import _timeindex_helper_func
 from edisgo.tools import tools
+from edisgo.tools.config import Config
 
 if TYPE_CHECKING:
     from edisgo import EDisGo
@@ -65,7 +65,7 @@ def oedb(
     dsm_cts = get_profile_cts(edisgo_obj, scenario, engine)
     ind_loads = edisgo_obj.topology.loads_df[
         (edisgo_obj.topology.loads_df.type == "conventional_load")
-        & (edisgo_obj.topology.loads_df.sector == "industry")
+        & (edisgo_obj.topology.loads_df.sector == "industrial")
     ]
     dsm_ind = get_profiles_per_industrial_load(
         ind_loads.building_id.unique(), scenario, engine
@@ -124,17 +124,27 @@ def get_profiles_per_industrial_load(
         column names are site ID as integer.
 
     """
-    saio.register_schema("demand", engine)
-    from saio.demand import (
-        egon_demandregio_sites_ind_electricity_dsm_timeseries as sites_ind_dsm_ts,
-    )
-    from saio.demand import (
+
+    config = Config()
+    (
+        sites_ind_dsm_ts,
         egon_osm_ind_load_curves_individual_dsm_timeseries,
         egon_sites_ind_load_curves_individual_dsm_timeseries,
+    ) = config.import_tables_from_oep(
+        engine,
+        [
+            "egon_demandregio_sites_ind_electricity_dsm_timeseries",
+            "egon_osm_ind_load_curves_individual_dsm_timeseries",
+            "egon_sites_ind_load_curves_individual_dsm_timeseries",
+        ],
+        "demand",
     )
 
     dsm_dict = {}
-
+    if len(load_ids) == 0:
+        for dsm_profile in ["e_min", "e_max", "p_min", "p_max"]:
+            dsm_dict[dsm_profile] = pd.DataFrame()
+        return dsm_dict
     with session_scope_egon_data(engine) as session:
         query = session.query(
             egon_sites_ind_load_curves_individual_dsm_timeseries.site_id,
@@ -160,7 +170,6 @@ def get_profiles_per_industrial_load(
             sites_ind_dsm_ts.scn_name == scenario,
             sites_ind_dsm_ts.industrial_sites_id.in_(load_ids),
         )
-
         df_sites_2 = pd.read_sql(sql=query.statement, con=engine)
 
     with session_scope_egon_data(engine) as session:
@@ -226,8 +235,10 @@ def get_profile_cts(
     egon_data and ding0.
 
     """
-    saio.register_schema("demand", engine)
-    from saio.demand import egon_etrago_electricity_cts_dsm_timeseries
+    config = Config()
+    (egon_etrago_electricity_cts_dsm_timeseries,) = config.import_tables_from_oep(
+        engine, ["egon_etrago_electricity_cts_dsm_timeseries"], "demand"
+    )
 
     # get data
     dsm_dict = {}

edisgo/io/electromobility_import.py

@@ -10,13 +10,13 @@
 
 import numpy as np
 import pandas as pd
-import saio
 
 from numpy.random import default_rng
 from sklearn import preprocessing
 from sqlalchemy.engine.base import Engine
 
 from edisgo.io.db import get_srid_of_db_table, session_scope_egon_data
+from edisgo.tools.config import Config
 
 if "READTHEDOCS" not in os.environ:
     import geopandas as gpd
@@ -1217,8 +1217,10 @@ def simbev_config_from_oedb(
         more information.
 
     """
-    saio.register_schema("demand", engine)
-    from saio.demand import egon_ev_metadata
+    config = Config()
+    (egon_ev_metadata,) = config.import_tables_from_oep(
+        engine, ["egon_ev_metadata"], "demand"
+    )
 
     with session_scope_egon_data(engine) as session:
         query = session.query(egon_ev_metadata).filter(
@@ -1252,8 +1254,10 @@ def potential_charging_parks_from_oedb(
         for more information.
 
     """
-    saio.register_schema("grid", engine)
-    from saio.grid import egon_emob_charging_infrastructure
+    config = Config()
+    (egon_emob_charging_infrastructure,) = config.import_tables_from_oep(
+        engine, ["egon_emob_charging_infrastructure"], "grid"
+    )
 
     crs = edisgo_obj.topology.grid_district["srid"]
 
@@ -1308,9 +1312,10 @@ def charging_processes_from_oedb(
         more information.
 
     """
-
-    saio.register_schema("demand", engine)
-    from saio.demand import egon_ev_mv_grid_district, egon_ev_trip
+    config = Config()
+    egon_ev_mv_grid_district, egon_ev_trip = config.import_tables_from_oep(
+        engine, ["egon_ev_mv_grid_district", "egon_ev_trip"], "demand"
+    )
 
     # get EV pool in grid
     scenario_variation = {"eGon2035": "NEP C 2035", "eGon100RE": "Reference 2050"}

edisgo/io/generators_import.py

@@ -8,13 +8,13 @@
 
 import numpy as np
 import pandas as pd
-import saio
 
 from sqlalchemy import func
 from sqlalchemy.engine.base import Engine
 
 from edisgo.io.db import get_srid_of_db_table, session_scope_egon_data
 from edisgo.tools import session_scope
+from edisgo.tools.config import Config
 from edisgo.tools.geo import find_nearest_bus, proj2equidistant
 from edisgo.tools.tools import (
     determine_bus_voltage_level,
@@ -937,11 +937,15 @@ def _get_egon_chp_plants():
             ).to_crs(srid_edisgo)
         return chp_gdf
 
-    saio.register_schema("supply", engine)
-    from saio.supply import (
+    config = Config()
+    (
         egon_chp_plants,
         egon_power_plants,
         egon_power_plants_pv_roof_building,
+    ) = config.import_tables_from_oep(
+        engine,
+        ["egon_chp_plants", "egon_power_plants", "egon_power_plants_pv_roof_building"],
+        "supply",
     )
 
     # get generator data from database

edisgo/io/heat_pump_import.py

@@ -4,11 +4,11 @@
 
 import numpy as np
 import pandas as pd
-import saio
 
 from sqlalchemy import func
 
 from edisgo.io import db
+from edisgo.tools.config import Config
 from edisgo.tools.tools import (
     determine_bus_voltage_level,
     determine_grid_integration_voltage_level,
@@ -267,24 +267,33 @@ def _get_individual_heat_pump_capacity():
         else:
             return np.sum(cap)
 
-    saio.register_schema("demand", engine)
-    from saio.demand import egon_district_heating_areas, egon_hp_capacity_buildings
-
-    saio.register_schema("supply", engine)
-    from saio.supply import (
+    config = Config()
+    (
+        egon_district_heating_areas,
+        egon_hp_capacity_buildings,
+    ) = config.import_tables_from_oep(
+        engine, ["egon_district_heating_areas", "egon_hp_capacity_buildings"], "demand"
+    )
+    (
         egon_district_heating,
         egon_era5_weather_cells,
         egon_individual_heating,
+    ) = config.import_tables_from_oep(
+        engine,
+        ["egon_district_heating", "egon_era5_weather_cells", "egon_individual_heating"],
+        "supply",
     )
-
-    saio.register_schema("boundaries", engine)
-    from saio.boundaries import (
+    (
         egon_map_zensus_mvgd_buildings,
         egon_map_zensus_weather_cell,
+    ) = config.import_tables_from_oep(
+        engine,
+        ["egon_map_zensus_mvgd_buildings", "egon_map_zensus_weather_cell"],
+        "boundaries",
+    )
+    egon_etrago_bus, egon_etrago_link = config.import_tables_from_oep(
+        engine, ["egon_etrago_bus", "egon_etrago_link"], "grid"
     )
-
-    saio.register_schema("grid", engine)
-    from saio.grid import egon_etrago_bus, egon_etrago_link
 
     building_ids = edisgo_object.topology.loads_df.building_id.unique()
     mv_grid_geom_srid = edisgo_object.topology.grid_district["srid"]
@@ -604,8 +613,10 @@ def efficiency_resistive_heaters_oedb(scenario, engine):
         given in p.u.
 
     """
-    saio.register_schema("scenario", engine)
-    from saio.scenario import egon_scenario_parameters
+    config = Config()
+    (egon_scenario_parameters,) = config.import_tables_from_oep(
+        engine, ["egon_scenario_parameters"], "scenario"
+    )
 
     # get cop from database
     with db.session_scope_egon_data(engine) as session:

edisgo/io/storage_import.py

@@ -6,11 +6,11 @@
 from typing import TYPE_CHECKING
 
 import pandas as pd
-import saio
 
 from sqlalchemy.engine.base import Engine
 
 from edisgo.io.db import session_scope_egon_data
+from edisgo.tools.config import Config
 from edisgo.tools.tools import (
     determine_bus_voltage_level,
     determine_grid_integration_voltage_level,
@@ -49,8 +49,10 @@ def home_batteries_oedb(
         units.
 
     """
-    saio.register_schema("supply", engine)
-    from saio.supply import egon_home_batteries
+    config = Config()
+    (egon_home_batteries,) = config.import_tables_from_oep(
+        engine, ["egon_home_batteries"], "supply"
+    )
 
     with session_scope_egon_data(engine) as session:
         query = (