Feature/extend steel example

.gitignore

@@ -7,3 +7,6 @@ __pycache__/
 /tests/_files/tabular_datapackage_hack_a_thon/data/
 /tests/_files/tabular_datapackage_hack_a_thon/datapackage.json
 /tests/_files/tabular_datapackage_mininmal_example/
+/tests/_files/tsam/*
+/examples/industry/collections/*
+/examples/industry/datapackage/*

data_adapter_oemof/adapters.py

@@ -4,7 +4,6 @@
 import itertools
 import json
 import logging
-import warnings
 from typing import Optional, Type, Union
 
 import pandas as pd
@@ -159,7 +158,10 @@ def get_data(self, key, field_type: Optional[Type] = None):
                     f"Using existing timeseries column '{timeseries_key}'."
                 )
                 return timeseries_key
-            logger.warning(f"Could not find timeseries entry for mapped key '{key}'")
+            logger.warning(
+                f"For Process {self.process_name}"
+                f"Could not find timeseries entry for mapped key '{key}'"
+            )
             return None
 
         # 2 Use defaults
@@ -228,11 +230,6 @@ def get_busses(self) -> dict:
                 struct = self.structure["default"]
             else:
                 struct = self.structure
-                warnings.warn(
-                    "Please check structure and provide either one set of inputs/outputs "
-                    "or specify as default Parameter specific busses not implemented yet. "
-                    f"No Bus found for Process {self.process_name} in Adapter {self}"
-                )
 
             # 2. Check for default busses
             if bus in ("bus", "from_bus", "to_bus", "fuel_bus"):
@@ -242,6 +239,11 @@ def get_busses(self) -> dict:
                     busses = struct["inputs"]
                 if bus == "to_bus":
                     busses = struct["outputs"]
+                if (
+                    self.__class__.type == "storage"
+                    and struct["inputs"] == struct["outputs"]
+                ):
+                    busses = struct["outputs"]
                 if len(busses) != 1:
                     raise MappingError(
                         f"Could not map {bus} to default bus - too many options"
@@ -426,6 +428,57 @@ def get_default_parameters(
         if self.get("carrier") == "carrier":
             defaults["carrier"] = self.get_busses()["bus"]
 
+        if self.get("amount") == None:
+            amount = 9999999999999
+            logger.warning(
+                f"Adding parameter 'amount' with value {amount} to commodity "
+                f"{self.process_name}. \n         "
+                f"This is beneficial if your commodity is functioning as "
+                f"shortage or unlimited import/source. "
+                f"Otherwise please add 'amount' to your commodity!"
+            )
+            defaults["amount"] = amount
+
+        return defaults
+
+
+class CommodityGHGAdapter(CommodityAdapter):
+    """
+    CommodityGHGAdapter
+    """
+
+    type = "commodity_ghg"
+    facade = facades.CommodityGHG
+
+    def get_busses(self) -> dict:
+        bus_list = self.structure["outputs"]
+        bus_dict = {}
+        counter = 0
+        for bus in bus_list:
+            if not bus.startswith("emi"):
+                bus_dict["bus"] = bus
+            elif bus.startswith("emi"):
+                bus_dict[f"emission_bus_{counter}"] = bus
+                counter += 1
+
+        # check that bus is defined
+        if bus_dict.get("bus") is None:
+            raise KeyError(f"{self.process_name} is missing 'bus', the input.")
+        return bus_dict
+
+    def get_default_parameters(self) -> dict:
+        defaults = super().get_default_parameters()
+        for key, value in self.data.items():
+            if key.startswith("ef"):
+                # adapt to the naming convention in oemof.tabular commodityGHG facade: emission_factor_<emission_bus_label>
+                target_label = None
+                emission_bus_labels = [key for item, key in defaults.items() if item.startswith("emission_bus")]
+                for label in emission_bus_labels:
+                    if label in key:
+                        target_label = label
+                if target_label == None:
+                    raise ValueError(f"Emission factor of {self.process_name} is named {key} but None of the emission buses matches: {emission_bus_labels}.")
+                defaults[f"emission_factor_{target_label}"] = value
         return defaults
 
 
@@ -439,6 +492,44 @@ class ConversionAdapter(Adapter):
     facade = facades.Conversion
 
 
+class ConversionGHGAdapter(Adapter):
+    """
+    ConversionGHGAdapter
+    """
+
+    type = "conversion_ghg"
+    facade = facades.ConversionGHG
+
+    def get_busses(self) -> dict:
+        def get_bus_from_struct(bus_list: list, bus_key: str) -> dict:
+            bus_dict = {}
+            counter = 0
+            for bus in bus_list:
+                if not bus.startswith("emi"):
+                    bus_dict[f"{bus_key}"] = bus
+                elif bus.startswith("emi"):
+                    bus_dict[f"emission_bus_{counter}"] = bus
+                    counter += 1
+            return bus_dict
+
+        return_bus_dict = get_bus_from_struct(
+            self.structure["inputs"], bus_key="from_bus"
+        ) | get_bus_from_struct(self.structure["outputs"], bus_key="to_bus")
+
+        # check that from_bus and to_bus is defined
+        for key in ["from_bus", "to_bus"]:
+            if return_bus_dict.get(key) is None:
+                raise KeyError(f"{self.process_name} is missing {key}.")
+        return return_bus_dict
+
+    def get_default_parameters(self) -> dict:
+        defaults = super().get_default_parameters()
+        for key, value in self.data.items():
+            if key.startswith("ef"):
+                defaults[key.replace("ef", "emission_factor")] = value
+        return defaults
+
+
 class LoadAdapter(Adapter):
     """
     LoadAdapter
@@ -492,6 +583,7 @@ class MIMOAdapter(Adapter):
         Field(name="region", type=str),
         Field(name="year", type=int),
         Field(name="groups", type=dict),
+        Field(name="lifetime", type=float),
         Field(name="capacity_cost", type=float),
         Field(name="capacity", type=float),
         Field(name="expandable", type=bool),
@@ -518,10 +610,13 @@ def get_default_parameters(self) -> dict:
             "emissions_factor_",
             "conversion_factor_",
             "flow_share_",
+            "ef_",
         )
         for key, value in self.data.items():
             for keyword in keywords:
                 if key.startswith(keyword):
+                    if key.startswith("ef"):
+                        key = key.replace("ef", "emission_factor")
                     defaults[key] = value
         return defaults
 

data_adapter_oemof/build_datapackage.py

@@ -1,4 +1,5 @@
 import dataclasses
+import logging
 import os
 import warnings
 from typing import Optional, Type
@@ -11,9 +12,12 @@
 
 from data_adapter_oemof.adapters import FACADE_ADAPTERS
 from data_adapter_oemof.adapters import Adapter as FacadeAdapter
+from data_adapter_oemof.calculations import handle_nans
 from data_adapter_oemof.settings import BUS_MAP, PARAMETER_MAP, PROCESS_ADAPTER_MAP
 from data_adapter_oemof.utils import convert_mixed_types_to_same_length
 
+logger = logging.getLogger()
+
 
 # Define a function to aggregate differing values into a list
 def _listify_to_periodic(group_df) -> pd.Series:
@@ -48,10 +52,11 @@ def _listify_to_periodic(group_df) -> pd.Series:
 
 
     """
-
+    group_df = handle_nans(group_df)
     if "year" not in group_df.columns:
         return group_df
     unique_values = pd.Series(dtype=object)
+
     for col in group_df.columns:
         if isinstance(group_df[col][group_df.index[0]], dict):
             # Unique input/output parameters are not allowed per period
@@ -64,21 +69,6 @@ def _listify_to_periodic(group_df) -> pd.Series:
         ):
             values = group_df[col].explode().unique()
         else:
-            # FIXME: Hotfix "if not" statement to replace nan values from lists:
-            #   in final data only complete datasets are expected.
-            if not all(group_df[col].isna()) and any(group_df[col].isna()):
-                group_df.loc[group_df[col].isna(), col] = (
-                    group_df[col]
-                    .dropna()
-                    .sample(
-                        group_df[col]
-                        .isna()
-                        .sum(),  # get the same number of values as are missing
-                        replace=True,
-                        random_state=0,
-                    )
-                    .values
-                )  # throw out the index
             values = group_df[col].unique()
         if len(values) > 1:
             if isinstance(group_df[col].iloc[0], list):
@@ -90,7 +80,6 @@ def _listify_to_periodic(group_df) -> pd.Series:
                 unique_values[col] = group_df[col].iat[0][0]
             else:
                 unique_values[col] = group_df[col].iat[0]
-    unique_values["name"] = "_".join(group_df.name)
     unique_values.drop("year")
     return unique_values
 
@@ -463,6 +452,7 @@ def build_datapackage(
         parameter_map: Optional[dict] = PARAMETER_MAP,
         bus_map: Optional[dict] = BUS_MAP,
         location_to_save_to: str = None,
+        debug=False,
     ):
         """
         Creating a Datapackage from the oemof_data_adapter that fits oemof.tabular Datapackages.
@@ -508,6 +498,9 @@ def _reduce_lists(x):
                     + _reduce_lists(timeseries.columns.get_level_values(1))
                 )
             facade_adapter_name: str = process_adapter_map[process_name]
+            logger.info(
+                f"Adaptering process {process_name} into adapter {facade_adapter_name}"
+            )
             facade_adapter: Type[FacadeAdapter] = FACADE_ADAPTERS[facade_adapter_name]
             component_adapter: Optional[FacadeAdapter] = None
             components = []
@@ -553,6 +546,22 @@ def _reduce_lists(x):
         )
         periods = cls.get_periods_from_parametrized_sequences(parametrized_sequences)
 
+        def reduce_data_frame(data_frame, steps=4):
+            """reduces `df` to 5 time steps per period"""
+            df = data_frame.copy()
+            df["ind"] = df.index
+            df["ind"] = df["ind"].apply(
+                lambda x: True if x.month == 1 and x.day == 1 and x.hour <= steps else False
+            )
+            df_reduced = df.loc[df["ind"] == 1].drop(columns=["ind"])
+            return df_reduced
+
+        if debug:
+            periods = reduce_data_frame(data_frame=periods)
+            for key, value in parametrized_sequences.items():
+                df_short = reduce_data_frame(data_frame=value)
+                parametrized_sequences.update({key: df_short})
+
         return cls(
             parametrized_elements=parametrized_elements,
             parametrized_sequences=parametrized_sequences,