Fix and extend tests

.github/workflows/pytest.yml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.8', '3.9', '3.10' ]
+        python-version: [ '3.9', '3.10' ]
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:

.gitignore

@@ -1,4 +1,4 @@
 .idea/
-collections/
+collections/*
 __pycache__/
-/structures/
+#/structures/

.pre-commit-config.yaml

@@ -23,18 +23,13 @@ repos:
         entry: black
         language: python
         types: [ python ]
+      - id: isort
+        name: isort
+        entry: isort
+        language: python
+        types: [ python ]
       - id: flake8
         name: flake8
         entry: flake8
         language: python
         types: [python]
-      - id: pylint
-        name: pylint
-        entry: pylint
-        language: system
-        types: [python]
-        args:
-          [
-            "-rn", # Only display messages
-            "-sn", # Don't display the score
-          ]

conftest.py

@@ -0,0 +1,6 @@
+import pathlib
+
+import dotenv
+
+TEST_ENV_FILE = pathlib.Path(__file__).parent / "tests" / ".env"
+dotenv.load_dotenv(TEST_ENV_FILE)

data_adapter_oemof/adapters.py

@@ -1,8 +1,6 @@
 import dataclasses
 import logging
 
-import oemof.solph
-import pandas
 from oemof.solph import Bus
 from oemof.tabular import facades
 

data_adapter_oemof/build_datapackage.py

@@ -21,7 +21,8 @@ def refactor_timeseries(timeseries: pd.DataFrame):
     Parameters
     ----------
     timeseries: pd.DataFrame
-        timeseries in parameter-model format (https://github.com/sedos-project/oedatamodel#oedatamodel-parameter)
+        timeseries in parameter-model format
+        (https://github.com/sedos-project/oedatamodel#oedatamodel-parameter)
 
     Returns
     pd.DataFrame Tabular form of timeseries for multiple periods of similar
@@ -31,7 +32,6 @@ def refactor_timeseries(timeseries: pd.DataFrame):
     """
     # Combine all time series into one DataFrame
     df_timeseries = pd.DataFrame()
-    timeseries_timesteps = []
     if timeseries.empty:
         return df_timeseries
     # Iterate over different time periods/years
@@ -107,14 +107,16 @@ def get_foreign_keys(struct: list, mapper: Mapper, components: list) -> list:
         struct: list
             Energy System structure defining input/outputs for Processes
         mapper: Mapper
-            for one element of the Process (foreign keys have to be equal for all components of a Process)
+            for one element of the Process
+            (foreign keys have to be equal for all components of a Process)
         components: list
-            all components as of a Process as dicts. Helps to check what columns that could be pointing to sequences
-            are found in Sequences.
+            all components as of a Process as dicts. Helps to check what columns
+            that could be pointing to sequences are found in Sequences.
 
         Returns
         -------
-        list of foreignKeys for Process including bus references and pointers to files containing `profiles`
+        list of foreignKeys for Process including bus references and pointers to files
+        containing `profiles`
         """
         new_foreign_keys = []
         components = pd.DataFrame(components)
@@ -139,9 +141,13 @@ def get_foreign_keys(struct: list, mapper: Mapper, components: list) -> list:
                         }
                     )
                 elif any(components[field.name].isin(mapper.timeseries.columns)):
+                    # Todo clean up on examples:
+                    #   -remove DE from hackerthon or
+                    #   -create propper example with realistic project data
                     warnings.warn(
                         "Not all profile columns are set within the given profiles."
-                        f" Please check if there is a timeseries for every Component in {mapper.process_name}"
+                        f" Please check if there is a timeseries for every Component in "
+                        f"{mapper.process_name}"
                     )
                     new_foreign_keys.append(
                         {
@@ -152,7 +158,9 @@ def get_foreign_keys(struct: list, mapper: Mapper, components: list) -> list:
                         }
                     )
                 else:
-                    # Most likely the field may be a Timeseries in this case, but it is a scalar or unused.
+                    # The Field is allowed to be a timeseries
+                    # -> and likely is a supposed to be a timeseries
+                    # but a scalar or `unused` is found.
                     pass
         return new_foreign_keys
 
@@ -165,8 +173,9 @@ def save_datapackage_to_csv(self, destination: str) -> None:
         self: DataPackage
             DataPackage to save
         destination: str
-            String to where the datapackage save to. More convenient to use os.path. If last level of folder stucture
-            does not exist, it will be created (as well as /elements and /sequences)
+            String to where the datapackage save to. More convenient to use os.path.
+            If last level of folder stucture does not exist, it will be created
+            (as well as /elements and /sequences)
 
         Returns
         -------
@@ -228,8 +237,8 @@ def build_datapackage(cls, adapter: Adapter):
         Parameters
         ----------
         adapter: Adapter
-            Adapter from oemof_data_adapter that is able to handle parameter model data from Databus.
-            Adapter needs to be initialized with `structure_name`. Use `links_
+            Adapter from oemof_data_adapter that is able to handle parameter model data
+            from Databus. Adapter needs to be initialized with `structure_name`. Use `links_
 
         Returns
         -------
@@ -267,8 +276,8 @@ def build_datapackage(cls, adapter: Adapter):
             parametrized_elements["bus"] += process_busses
 
             # getting foreign keys with last component
-            #   foreign keys have to be equal for every component within a Process as foreign key columns cannot
-            #   have mixed meaning
+            #   foreign keys have to be equal for every component within a Process
+            #   as foreign key columns cannot have mixed meaning
             foreign_keys[process_name] = cls.get_foreign_keys(
                 struct, component_mapper, components
             )

data_adapter_oemof/mappings.py

@@ -135,9 +135,10 @@ def get_busses(self, struct):
             If found, search for similarities in the structure CSV.
             If not, search for name similarities:
                 Between the structure CSV and the adapter's buses take name from the structure.
-        Note: If passed class has more than two busses or different names for busses fields it is highly recommended
-        to provide BUS_NAME_MAP entry for this class. If multiple instances of the same facade
-        shall be having different inputs/outputs a facade Adapter has to be added for each.
+        Note: If passed class has more than two busses or different names for busses fields it
+        is highly recommended to provide BUS_NAME_MAP entry for this class.
+        If multiple instances of the same facade shall be having different inputs/outputs
+        a facade Adapter has to be added for each.
         :param struct: dict
         :return: dictionary with tabular like Busses
         """
@@ -153,7 +154,6 @@ def get_busses(self, struct):
 
         bus_dict = {}
         for bus in bus_occurrences_in_fields:  # emission_bus
-
             # 1. Check for existing mappings
             try:
                 bus_dict[bus] = self.bus_map[self.adapter.__name__][bus]
@@ -169,8 +169,8 @@ def get_busses(self, struct):
                 struct = struct["default"]
             else:
                 warnings.warn(
-                    "Please check structure and provide either one set of inputs/outputs or specify as default"
-                    "Parameter specific busses not implemented yet"
+                    "Please check structure and provide either one set of inputs/outputs "
+                    "or specify as default Parameter specific busses not implemented yet"
                 )
 
             # 2. Check for default busses
@@ -183,7 +183,7 @@ def get_busses(self, struct):
                     busses = struct["outputs"]
                 if len(busses) != 1:
                     raise MappingError(
-                        f"Could not map {bus=} to default bus - too many options"
+                        f"Could not map {bus} to default bus - too many options"
                     )
                 bus_dict[bus] = busses[0]
                 continue

data_adapter_oemof/mappings/BUS_NAME_MAP.yaml

@@ -6,5 +6,5 @@ VolatileAdapter:
   bus: electricity
 
 ConversionAdapter:
-  from_bus: auto
-  to_bus: auto
+  from_bus: ch4
+  to_bus: electricity

data_adapter_oemof/mappings/PROCESS_TYPE_MAP.yaml

@@ -1,4 +1,4 @@
 # This is a mapping file to map the various processes to the oemof tabular facades.
 modex_tech_wind_turbine_onshore: VolatileAdapter
 modex_tech_storage_battery: StorageAdapter
-modex_tech_generator_gas: ExtractionTurbineAdapter
+modex_tech_generator_gas: ConversionAdapter

data_adapter_oemof/settings.py

@@ -1,4 +1,27 @@
+import os
 import pathlib
 
-ROOT_DIR = pathlib.Path(__file__).parent.parent
-COLLECTIONS_DIR = ROOT_DIR / "collections"
+ROOT_DIR = pathlib.Path(__file__).parent
+
+COLLECTIONS_DIR = (
+    pathlib.Path(os.environ["COLLECTIONS_DIR"])
+    if "COLLECTIONS_DIR" in os.environ
+    else pathlib.Path.cwd() / "collections"
+)
+if not COLLECTIONS_DIR.exists():
+    raise FileNotFoundError(
+        f"Could not find collections directory '{COLLECTIONS_DIR}'. "
+        "You should either create the collections folder or "
+        "change path to collection folder by changing environment variable 'COLLECTIONS_DIR'.",
+    )
+STRUCTURES_DIR = (
+    pathlib.Path(os.environ["STRUCTURES_DIR"])
+    if "STRUCTURES_DIR" in os.environ
+    else pathlib.Path.cwd() / "structures"
+)
+if not STRUCTURES_DIR.exists():
+    raise FileNotFoundError(
+        f"Could not find structure directory '{STRUCTURES_DIR}'. "
+        "You should either create the structure folder or "
+        "change path to structure folder by changing environment variable 'STRUCTURES_DIR'.",
+    )

pyproject.toml

@@ -1,5 +1,5 @@
 [tool.poetry]
-name = "data_adapter_oemof"
+name = "data_adapter_oemof_tests"
 version = "0.0.1dev"
 description = "Data adapter from oedatamodel to oemof.tabular datapackages"
 authors = [
@@ -8,16 +8,19 @@ authors = [
   "Julian Endres <[email protected]>",
 ]
 
+[tool.poetry.group.dev.dependencies]
+isort = "^5.12.0"
+
 [build-system]
 requires = ["poetry_core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<3.11"
-data_adapter = { git = "https://[email protected]/sedos-project/data_adapter", branch = "dev"}
+data_adapter = { git = "https://[email protected]/sedos-project/data_adapter", branch = "main"}
 oemof-tabular = { git = "https://[email protected]/oemof/oemof-tabular", tag = "v0.0.4.dev0"}
-# fix boto3 to fasten up dependency resolution
-boto3 = "1.26.125"
+boto3 = "1.26.125"  # fix boto3 to fasten up dependency resolution
+python-dotenv = "^0.21.0"
 
 [tool.poetry.dev-dependencies]
 black = "20.8b1"
@@ -43,3 +46,6 @@ exclude = '''
 )/
 '''
 max-line-length=100
+
+[tool.isort]
+profile = "black"

tests/.env

@@ -0,0 +1,3 @@
+
+COLLECTIONS_DIR=tests/collections
+STRUCTURES_DIR=tests/structures

tests/_files/build_datapackage_goal/data/elements/bus.csv

@@ -0,0 +1,3 @@
+name;type;balanced
+electricity;bus;True
+ch4;bus;True

tests/_files/build_datapackage_goal/data/elements/modex_tech_generator_gas.csv

@@ -0,0 +1,4 @@
+from_bus;to_bus;carrier;tech;capacity;efficiency;marginal_cost;carrier_cost;capacity_cost;expandable;capacity_potential;capacity_minimum;input_parameters;output_parameters;name;type
+ch4;electricity;gas;generator_gas;;1;0;0;;False;inf;;{};{};BB-2016-generator_gas;conversion
+ch4;electricity;gas;generator_gas;;1;0;0;;False;inf;;{};{};BB-2030-generator_gas;conversion
+ch4;electricity;gas;generator_gas;;1;0;0;;False;inf;;{};{};BB-2050-generator_gas;conversion

tests/_files/build_datapackage_goal/data/elements/modex_tech_storage_battery.csv

@@ -0,0 +1,4 @@
+bus;carrier;tech;storage_capacity;capacity;capacity_cost;storage_capacity_cost;storage_capacity_potential;capacity_potential;expandable;lifetime;age;fixed_costs;marginal_cost;efficiency;input_parameters;output_parameters;name;type
+electricity;Lithium;storage_battery;0;17.8;0;;inf;inf;False;;0;[];0;1;{};{};BB-2016-storage_battery;storage
+electricity;Lithium;storage_battery;0;17.8;0;;inf;inf;False;;0;[];0;1;{};{};BB-2030-storage_battery;storage
+electricity;Lithium;storage_battery;0;17.8;0;;inf;inf;False;;0;[];0;1;{};{};BB-2050-storage_battery;storage

tests/_files/build_datapackage_goal/data/elements/modex_tech_wind_turbine_onshore.csv

@@ -0,0 +1,4 @@
+bus;carrier;tech;profile;capacity;capacity_potential;capacity_minimum;expandable;marginal_cost;capacity_cost;lifetime;age;fixed_costs;output_parameters;name;type
+electricity;wind;wind_turbine_onshore;onshore_BB;5700.03;inf;;False;0;;25.4;0;23280.0;{};BB-2016-wind_turbine_onshore;volatile
+electricity;wind;wind_turbine_onshore;onshore_BB;5700.03375;inf;;False;0;;30.0;0;12600.0;{};BB-2030-wind_turbine_onshore;volatile
+electricity;wind;wind_turbine_onshore;onshore_BB;5700.03375;inf;;False;0;;30.0;0;11340.0;{};BB-2050-wind_turbine_onshore;volatile

tests/_files/build_datapackage_goal/data/sequences/modex_tech_wind_turbine_onshore_sequence.csv

@@ -0,0 +1,10 @@
+timeindex;onshore_BB
+2016-01-01T00:00:00;0.0516
+2016-01-01T01:00:00;0.051
+2016-01-01T02:00:00;0.0444
+2030-01-01T00:00:00;0.0526
+2030-01-01T01:00:00;0.051
+2030-01-01T02:00:00;0.0444
+2050-01-01T00:00:00;0.0536
+2050-01-01T01:00:00;0.051
+2050-01-01T02:00:00;0.0444