From f5043b4f6243dcfc06de0ca468dbb24ee7f0a337 Mon Sep 17 00:00:00 2001 From: Hendrik Huyskens Date: Wed, 16 Aug 2023 12:01:46 +0200 Subject: [PATCH 1/9] Add empty collections folder to tests --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 17e1ca2..1b73087 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .idea/ collections/* +tests/collections/ __pycache__/ -#/structures/ +/structures/ From 6e07bf215bd8af3d4409b044d0fd881bcbb0caca Mon Sep 17 00:00:00 2001 From: Hendrik Huyskens Date: Wed, 16 Aug 2023 12:02:46 +0200 Subject: [PATCH 2/9] Refactor Adapter to not init facade --- data_adapter_oemof/adapters.py | 134 +++++++++------------------------ data_adapter_oemof/mappings.py | 4 +- tests/test_data_adapter.py | 111 +++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 99 deletions(-) create mode 100644 tests/test_data_adapter.py diff --git a/data_adapter_oemof/adapters.py b/data_adapter_oemof/adapters.py index 4cb602f..67e222f 100644 --- a/data_adapter_oemof/adapters.py +++ b/data_adapter_oemof/adapters.py @@ -1,8 +1,7 @@ -import dataclasses import logging -from oemof.solph import Bus from oemof.tabular import facades +from oemof.tabular._facade import Facade from data_adapter_oemof import calculations from data_adapter_oemof.mappings import Mapper @@ -12,40 +11,23 @@ class Adapter: type: str = "adapter" - extra_attributes = ("name", "type", "year") - - def as_dict(self): - """ - Adds function to return DataFrame from adapter. - - This mixin is needed because `pd.DataFrame(dataclass_instance)` creates columns - only for existing dataclass attributes. - But we add custom_attributes (i.e. "name") which would be neglected. - """ - fields = dataclasses.fields(self) - data = {field.name: getattr(self, field.name) for field in fields} - for attr in self.extra_attributes: - data[attr] = getattr(self, attr) - return data - - @classmethod - def parametrize_dataclass( - cls, - struct, - mapper: Mapper, - ) -> "Adapter": - return cls(**cls.get_default_parameters(struct, mapper)) - - @classmethod - def get_default_parameters(cls, struct: dict, mapper: Mapper) -> dict: + facade: Facade = None + extra_attributes = ("name", "type") + + def as_dict(self) -> dict: + return self.facade_dict + + def __init__(self, struct: dict, mapper: Mapper): + self.facade_dict = self.get_default_parameters(struct, mapper) + + def get_default_parameters(self, struct: dict, mapper: Mapper) -> dict: defaults = { - "type": cls.type, + "type": self.type, } # Add mapped attributes mapped_values = mapper.get_default_mappings(struct) defaults.update(mapped_values) # Add additional attributes - attributes = { "region": mapper.get("region"), "year": mapper.get("year"), @@ -67,146 +49,104 @@ def get_default_parameters(cls, struct: dict, mapper: Mapper) -> dict: return defaults -def facade_adapter(cls): - r""" - Sets type of Busses to str - Sets 'build_solph_components' to False in __init__. - """ - # set type of bus to str - for field in dataclasses.fields(cls): - if field.type == Bus: - field.type = str - - original_init = cls.__init__ - - # do not build solph components - def new_init(self, *args, **kwargs): - custom_attributes = {} - original_fields = tuple(field.name for field in dataclasses.fields(cls)) - for key in tuple(kwargs.keys()): - if key not in original_fields: - custom_attributes[key] = kwargs.pop(key) - - original_init(self, *args, build_solph_components=False, **kwargs) - - for key, value in custom_attributes.items(): - setattr(self, key, value) - - cls.__init__ = new_init - - return cls - - -@facade_adapter -class DispatchableAdapter(facades.Dispatchable, Adapter): +class DispatchableAdapter(Adapter): """ Dispatchable Adapter """ - type = "dispatchable" + facade = facades.Dispatchable -@facade_adapter -class HeatPumpAdapter(facades.HeatPump, Adapter): +class HeatPumpAdapter(Adapter): """ HeatPump Adapter """ - type = "heat_pump" + facade = facades.HeatPump -@facade_adapter -class LinkAdapter(facades.Link, Adapter): +class LinkAdapter(Adapter): """ Link Adapter """ - type = "link" + facade = facades.Link -@facade_adapter -class ReservoirAdapter(facades.Reservoir, Adapter): +class ReservoirAdapter(Adapter): """ Reservoir Adapter """ - type = "reservoir" + facade = facades.Reservoir -@facade_adapter -class ExcessAdapter(facades.Excess, Adapter): +class ExcessAdapter(Adapter): """ Excess Adapter """ - type = "excess" + facade = facades.Excess -@facade_adapter -class BackpressureTurbineAdapter(facades.BackpressureTurbine, Adapter): +class BackpressureTurbineAdapter(Adapter): """ BackpressureTurbine Adapter """ - type = "backpressure_turbine" + facade = facades.BackpressureTurbine -@facade_adapter -class CommodityAdapter(facades.Commodity, Adapter): +class CommodityAdapter(Adapter): """ CommodityAdapter """ - type = "commodity" + facade = facades.Commodity -@facade_adapter -class ConversionAdapter(facades.Conversion, Adapter): +class ConversionAdapter(Adapter): """ ConversionAdapter To use Conversion, map the inputs and outputs within the structure to avoid deduction failure. """ - type = "conversion" + facade = facades.Conversion -@facade_adapter -class LoadAdapter(facades.Load, Adapter): +class LoadAdapter(Adapter): """ LoadAdapter """ - type = "load" + facade = facades.Load -@facade_adapter -class StorageAdapter(facades.Storage, Adapter): +class StorageAdapter(Adapter): """ StorageAdapter """ - type = "storage" + facade = facades.Storage -@facade_adapter -class ExtractionTurbineAdapter(facades.ExtractionTurbine, Adapter): +class ExtractionTurbineAdapter(Adapter): """ ExtractionTurbineAdapter """ - type = "extraction_turbine" + facade = facades.ExtractionTurbine -@facade_adapter -class VolatileAdapter(facades.Volatile, Adapter): +class VolatileAdapter(Adapter): """ VolatileAdapter """ - type = "volatile" + facade = facades.Volatile # Create a dictionary of all adapter classes defined in this module FACADE_ADAPTERS = { - name: adapter for name, adapter in globals().items() if name.endswith("Adapter") + adapter.type: adapter for name, adapter in globals().items() if name.endswith("Adapter") } diff --git a/data_adapter_oemof/mappings.py b/data_adapter_oemof/mappings.py index 8b92027..87f436f 100644 --- a/data_adapter_oemof/mappings.py +++ b/data_adapter_oemof/mappings.py @@ -144,7 +144,7 @@ def get_busses(self, struct): """ bus_occurrences_in_fields = [ field.name - for field in dataclasses.fields(self.adapter) + for field in dataclasses.fields(self.adapter.facade) if "bus" in field.name ] if len(bus_occurrences_in_fields) == 0: @@ -213,7 +213,7 @@ def get_default_mappings(self, struct): mapped_all_class_fields = { field.name: value - for field in dataclasses.fields(self.adapter) + for field in dataclasses.fields(self.adapter.facade) if (value := self.get(field.name, field.type)) is not None } mapped_all_class_fields.update(self.get_busses(struct)) diff --git a/tests/test_data_adapter.py b/tests/test_data_adapter.py new file mode 100644 index 0000000..e1b23ef --- /dev/null +++ b/tests/test_data_adapter.py @@ -0,0 +1,111 @@ +import pathlib +from collections import namedtuple + +import pandas as pd + +from data_adapter_oemof.mappings import Mapper +from data_adapter_oemof.adapters import VolatileAdapter, FACADE_ADAPTERS + +TEST_DIR = pathlib.Path(__file__).parent +TEMP_DIR = TEST_DIR / "_temp" + + +def test_simple_adapter(): + data = { + "type": "volatile", + "region": "B", + "carrier": "wind", + "bus": "electricity", + "tech": "onshore", + "capacity": 12, + "sedos-marginal_cost": 1, + "sedos-overnight_cost": 1000, + "capital_costs": 100, + "sedos-lifetime": 25, + "sedos-wacc": 0.05, + } + timeseries = pd.DataFrame({"a": [0, 0, 0], "onshore_B": [1, 2, 3]}) + struct = {"default": {"inputs": ["onshore"], "outputs": ["electricity"]}} + mapping = { + "modex_tech_wind_turbine_onshore": {"profile": "onshore", "lifetime": "sedos-lifetime"} + } + mapper = Mapper( + adapter=VolatileAdapter, + process_name="modex_tech_wind_turbine_onshore", + data=data, + timeseries=timeseries, + mapping=mapping, + ) + adapter = VolatileAdapter(struct, mapper) + expected_dict = { + "bus": "electricity", + "capacity": 12, + "carrier": "wind", + "name": "B-None-onshore", + "profile": "onshore_B", + "tech": "onshore", + "type": "volatile", + "lifetime": 25, + "region": "B", + "year": None + } + assert adapter.as_dict() == expected_dict + + +def test_adapter(): + Component = namedtuple("Component", ["type", "data"]) + + # first, a list of components and some data + components = [ + Component( + type="volatile", + data={ + "type": "volatile", + "region": "B", + "carrier": "wind", + "tech": tech, + "capacity": 12, + "sedos-marginal_cost": 1, + "sedos-overnight_cost": 1000, + "capital_costs": 100, + "sedos-lifetime": 25, + "sedos-wacc": 0.05, + }, + ) + for tech in ["onshore", "offshore"] + ] + + struct = {"default": {"inputs": ["onshore"], "outputs": ["electricity"]}} + + # collect instances of the adapters + parametrized = {} + for component in components: + if component.type not in parametrized: + parametrized[component.type] = [] + + adapter = TYPE_MAP[component.type] + + parametrized[component.type].append( + adapter.parametrize_dataclass(component.data, struct, None) + ) + # create a dictionary of dataframes + dataframes = { + type: pd.DataFrame([adapted.as_dict() for adapted in adapted_list]) + for type, adapted_list in parametrized.items() + } + for typ, df in dataframes.items(): + + path_default = ( + TEST_DIR + / "_files" + / "tabular_datapackage_mininmal_example" + / "data" + / "elements" + / f"{typ}.csv" + ) + + df_default = pd.read_csv(path_default, sep=";") + + assert set(df.columns) == set(df_default.columns) + + pd.testing.assert_frame_equal(df, df_default) From 4ea35fea3bc3019f299e62dcae552e5633982e50 Mon Sep 17 00:00:00 2001 From: "Julian.Endres" Date: Mon, 21 Aug 2023 18:47:15 +0200 Subject: [PATCH 3/9] Update lock file [data-adapter] --- poetry.lock | 91 ++++++++++++++++++++++------------------------------- 1 file changed, 37 insertions(+), 54 deletions(-) diff --git a/poetry.lock b/poetry.lock index b30f678..88f49c3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -498,7 +498,7 @@ test-randomorder = ["pytest-randomly"] [[package]] name = "data-adapter" -version = "0.11.1" +version = "0.11.2" description = "Provides general functionality for other data adapters" optional = false python-versions = ">=3.8.1,<4.0.0" @@ -516,8 +516,8 @@ sqlalchemy = "^1.4.46" [package.source] type = "git" url = "https://git@github.com/sedos-project/data_adapter" -reference = "main" -resolved_reference = "eba77f2866b2374830de29381dc028e02c383279" +reference = "dev" +resolved_reference = "2e81c2a145123023e1a863bf835255611775f0af" [[package]] name = "datapackage" @@ -744,13 +744,13 @@ test = ["objgraph", "psutil"] [[package]] name = "identify" -version = "2.5.26" +version = "2.5.27" description = "File identification library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.5.26-py2.py3-none-any.whl", hash = "sha256:c22a8ead0d4ca11f1edd6c9418c3220669b3b7533ada0a0ffa6cc0ef85cf9b54"}, - {file = "identify-2.5.26.tar.gz", hash = "sha256:7243800bce2f58404ed41b7c002e53d4d22bcf3ae1b7900c2d7aefd95394bf7f"}, + {file = "identify-2.5.27-py2.py3-none-any.whl", hash = "sha256:fdb527b2dfe24602809b2201e033c2a113d7bdf716db3ca8e3243f735dcecaba"}, + {file = "identify-2.5.27.tar.gz", hash = "sha256:287b75b04a0e22d727bc9a41f0d4f3c1bcada97490fa6eabb5b28f0e9097e733"}, ] [package.extras] @@ -879,23 +879,6 @@ files = [ [package.dependencies] six = "*" -[[package]] -name = "isort" -version = "5.12.0" -description = "A Python utility / library to sort Python imports." -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, - {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, -] - -[package.extras] -colors = ["colorama (>=0.4.3)"] -pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] -plugins = ["setuptools"] -requirements-deprecated-finder = ["pip-api", "pipreqs"] - [[package]] name = "jinja2" version = "3.1.2" @@ -1378,12 +1361,12 @@ files = [ [[package]] name = "petl" -version = "1.7.13" +version = "1.7.14" description = "A Python package for extracting, transforming and loading tables of data." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ - {file = "petl-1.7.13.tar.gz", hash = "sha256:964136f10582a511807929fae9854af229a67a972f64cfc7c636a65efca157d7"}, + {file = "petl-1.7.14.tar.gz", hash = "sha256:d4802e3c4804bf85f2267a0102fcad35c61e6a37c90d9e1a1674331f35a90a7f"}, ] [package.extras] @@ -1419,13 +1402,13 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co [[package]] name = "pluggy" -version = "1.2.0" +version = "1.3.0" description = "plugin and hook calling mechanisms for python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, - {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, ] [package.extras] @@ -1536,27 +1519,27 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] [[package]] name = "pyomo" -version = "6.6.1" +version = "6.6.2" description = "Pyomo: Python Optimization Modeling Objects" optional = false python-versions = ">=3.7" files = [ - {file = "Pyomo-6.6.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:ca93f624c536bd043cfa9dad7426cc8e317b95eb854d1f9796a4629fa8c37830"}, - {file = "Pyomo-6.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5ee14bdbfbe532379329d0e50306bdc609feb548b5e77c2d51b43077b6bbd7c"}, - {file = "Pyomo-6.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:32644ad06a962917bcc1246ebbdc825b30342ea8cada39e01ef483d1eb60456c"}, - {file = "Pyomo-6.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fa2442abdd4872320c0a8eca960136fdd9d4faa14353be3391eff11403547027"}, - {file = "Pyomo-6.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f795a4fb345ab5676acf57b39643a3605f1d88c6d10d6710514d959bd52bfb1"}, - {file = "Pyomo-6.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:44e0b12f82f0c70ed246bb8b5eed8279cee07d7f2218bc8dd955562b9a82e595"}, - {file = "Pyomo-6.6.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:05764117cecff51897ef473b000afbb6b353f51b70ce2101f16c9c447c8c0d31"}, - {file = "Pyomo-6.6.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96302d16a5fa94e6985d388fc42d7bfbcddd6f65a165edf92497bdc40bd33bec"}, - {file = "Pyomo-6.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1283eb424a4f23eeefb82716124dc43611a6131e3a1cfee45c7bd7d3901ad392"}, - {file = "Pyomo-6.6.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:2a6311679f60d1e2f3e611029d4436188e4873e33dcc7f110426e5b96248164f"}, - {file = "Pyomo-6.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b14a4b72f9a10cc28303ba9625f493cf19d4398d91632b7bff5bd36f6bfe670"}, - {file = "Pyomo-6.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:f9b99bbbc9cada3af590627a1daba21676f7c63f55f957c8025f5da3c8cca7fd"}, - {file = "Pyomo-6.6.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:d27678616ea4e619554b158a9321f3d71702504ed6b0df4f3d38c2a2940a6eae"}, - {file = "Pyomo-6.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:024fec287be5bc7bde660888f9f9309f71cb76dfef99a2b61ec286045a2c03a3"}, - {file = "Pyomo-6.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:f32de1a02f35bf3888229d3ddee3e2e176f3de8ec6d506d2660830794991afa2"}, - {file = "Pyomo-6.6.1.tar.gz", hash = "sha256:3fb0aba7b0f4120e6ce0f242502c0e61478d61e326bc90b7dc392bbefd114b34"}, + {file = "Pyomo-6.6.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:a6bf4c8e8560b03a23fee783ea425c01abfe7b17544a8011fed5da09e9253615"}, + {file = "Pyomo-6.6.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8508e8cf654ada06675da2ae4291935f0ba1c8bd44285d133b1bc2e2ce0350f1"}, + {file = "Pyomo-6.6.2-cp310-cp310-win_amd64.whl", hash = "sha256:c866666093c055a9956dabee42b876e9533eba8703d892d73945620b252e333c"}, + {file = "Pyomo-6.6.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cd2e262b5d2e754b47c2f376c7ff0f48dff74f2fdad08b995db0b1539553c64d"}, + {file = "Pyomo-6.6.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95fe255dd402af84e1294dc22437a2c292ef708f04974da0aa56eb10065178f1"}, + {file = "Pyomo-6.6.2-cp311-cp311-win_amd64.whl", hash = "sha256:6a2463b38bbe1f8e50584ba87659ec425e420236070b078b707849fac7fb6b02"}, + {file = "Pyomo-6.6.2-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:cdd2bc12a0e536d3cc5a5535efaf8af8499d12e790b564e46897d68092e335b3"}, + {file = "Pyomo-6.6.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e3fe82aaabb7f5aa0ab60f4fb05ba038dee014f9dfc10657d1b6cd0e5910f90"}, + {file = "Pyomo-6.6.2-cp37-cp37m-win_amd64.whl", hash = "sha256:067c2f533bb0f05b673f95caa38f6b6d3d2f774a3b97aca280b1350a1516857f"}, + {file = "Pyomo-6.6.2-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:f26cb95531580fb4932cad04962aca854bc7bd3832c77f66ef7c502891c59b6f"}, + {file = "Pyomo-6.6.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c94e2a711f59ac81bd576955529944b286a3fd87cdfe95011534966746af289"}, + {file = "Pyomo-6.6.2-cp38-cp38-win_amd64.whl", hash = "sha256:47402204e99df3fefd6ee584b45cfca4b00f60a1dce20208340449b152b308a2"}, + {file = "Pyomo-6.6.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:9a739bc8add47c1a6275746d272c6e664d0240923a9b6fd58800173e3f513a12"}, + {file = "Pyomo-6.6.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:867d6351718e8d0e14b5144a83a964348fd0d16dd5d19db429296349f17755c9"}, + {file = "Pyomo-6.6.2-cp39-cp39-win_amd64.whl", hash = "sha256:e22380fe8615b8e29cc4e01dd3b08ce9575867242fe4fb974589dfd93f00926b"}, + {file = "Pyomo-6.6.2.tar.gz", hash = "sha256:c8ad55213ff8b1a2c4e469110db8079722d5a6f364c6c46a42e2f750fc9e4d26"}, ] [package.dependencies] @@ -1899,29 +1882,29 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] [[package]] name = "setuptools" -version = "68.1.0" +version = "68.1.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-68.1.0-py3-none-any.whl", hash = "sha256:e13e1b0bc760e9b0127eda042845999b2f913e12437046e663b833aa96d89715"}, - {file = "setuptools-68.1.0.tar.gz", hash = "sha256:d59c97e7b774979a5ccb96388efc9eb65518004537e85d52e81eaee89ab6dd91"}, + {file = "setuptools-68.1.2-py3-none-any.whl", hash = "sha256:3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b"}, + {file = "setuptools-68.1.2.tar.gz", hash = "sha256:3d4dfa6d95f1b101d695a6160a7626e15583af71a5f52176efa5d39a054d475d"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5,<=7.1.2)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "shellingham" -version = "1.5.2" +version = "1.5.3" description = "Tool to Detect Surrounding Shell" optional = false python-versions = ">=3.7" files = [ - {file = "shellingham-1.5.2-py2.py3-none-any.whl", hash = "sha256:db9b385e903ebfe07958f6df493d47d4432ff5bed71e75ed3e613ace090be5f3"}, - {file = "shellingham-1.5.2.tar.gz", hash = "sha256:95946024df2db98c83382606a9ae875f613b15c950c980a3bf7a5adde40e7720"}, + {file = "shellingham-1.5.3-py2.py3-none-any.whl", hash = "sha256:419c6a164770c9c7cfcaeddfacb3d31ac7a8db0b0f3e9c1287679359734107e9"}, + {file = "shellingham-1.5.3.tar.gz", hash = "sha256:cb4a6fec583535bc6da17b647dd2330cf7ef30239e05d547d99ae3705fd0f7f8"}, ] [[package]] @@ -2325,4 +2308,4 @@ docs = [] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.11" -content-hash = "405d0685bb24e4746de2f4273703e9019d180fc68a0a13529859a37b5ecac55c" +content-hash = "6c98ac20f1f11d8ad34518a8a7eb949de6c85e26a566dd1408c10f3d7e6d05f0" From aada73ee4c482ee846df6302d728ba8c83b857d5 Mon Sep 17 00:00:00 2001 From: FelixMau Date: Mon, 28 Aug 2023 18:29:03 +0200 Subject: [PATCH 4/9] Update PROCESS_TYPE_MAP.yaml No data_adapter test since it should be tested in data_adapter itself --- .../mappings/PROCESS_TYPE_MAP.yaml | 6 +- tests/test_data_adapter.py | 222 +++++++++--------- 2 files changed, 114 insertions(+), 114 deletions(-) diff --git a/data_adapter_oemof/mappings/PROCESS_TYPE_MAP.yaml b/data_adapter_oemof/mappings/PROCESS_TYPE_MAP.yaml index f24b43b..7906e83 100644 --- a/data_adapter_oemof/mappings/PROCESS_TYPE_MAP.yaml +++ b/data_adapter_oemof/mappings/PROCESS_TYPE_MAP.yaml @@ -1,4 +1,4 @@ # This is a mapping file to map the various processes to the oemof tabular facades. -modex_tech_wind_turbine_onshore: VolatileAdapter -modex_tech_storage_battery: StorageAdapter -modex_tech_generator_gas: ConversionAdapter +modex_tech_wind_turbine_onshore: volatile +modex_tech_storage_battery: storage +modex_tech_generator_gas: conversion diff --git a/tests/test_data_adapter.py b/tests/test_data_adapter.py index e1b23ef..4f5cc5b 100644 --- a/tests/test_data_adapter.py +++ b/tests/test_data_adapter.py @@ -1,111 +1,111 @@ -import pathlib -from collections import namedtuple - -import pandas as pd - -from data_adapter_oemof.mappings import Mapper -from data_adapter_oemof.adapters import VolatileAdapter, FACADE_ADAPTERS - -TEST_DIR = pathlib.Path(__file__).parent -TEMP_DIR = TEST_DIR / "_temp" - - -def test_simple_adapter(): - data = { - "type": "volatile", - "region": "B", - "carrier": "wind", - "bus": "electricity", - "tech": "onshore", - "capacity": 12, - "sedos-marginal_cost": 1, - "sedos-overnight_cost": 1000, - "capital_costs": 100, - "sedos-lifetime": 25, - "sedos-wacc": 0.05, - } - timeseries = pd.DataFrame({"a": [0, 0, 0], "onshore_B": [1, 2, 3]}) - struct = {"default": {"inputs": ["onshore"], "outputs": ["electricity"]}} - mapping = { - "modex_tech_wind_turbine_onshore": {"profile": "onshore", "lifetime": "sedos-lifetime"} - } - mapper = Mapper( - adapter=VolatileAdapter, - process_name="modex_tech_wind_turbine_onshore", - data=data, - timeseries=timeseries, - mapping=mapping, - ) - adapter = VolatileAdapter(struct, mapper) - expected_dict = { - "bus": "electricity", - "capacity": 12, - "carrier": "wind", - "name": "B-None-onshore", - "profile": "onshore_B", - "tech": "onshore", - "type": "volatile", - "lifetime": 25, - "region": "B", - "year": None - } - assert adapter.as_dict() == expected_dict - - -def test_adapter(): - Component = namedtuple("Component", ["type", "data"]) - - # first, a list of components and some data - components = [ - Component( - type="volatile", - data={ - "type": "volatile", - "region": "B", - "carrier": "wind", - "tech": tech, - "capacity": 12, - "sedos-marginal_cost": 1, - "sedos-overnight_cost": 1000, - "capital_costs": 100, - "sedos-lifetime": 25, - "sedos-wacc": 0.05, - }, - ) - for tech in ["onshore", "offshore"] - ] - - struct = {"default": {"inputs": ["onshore"], "outputs": ["electricity"]}} - - # collect instances of the adapters - parametrized = {} - for component in components: - if component.type not in parametrized: - parametrized[component.type] = [] - - adapter = TYPE_MAP[component.type] - - parametrized[component.type].append( - adapter.parametrize_dataclass(component.data, struct, None) - ) - # create a dictionary of dataframes - dataframes = { - type: pd.DataFrame([adapted.as_dict() for adapted in adapted_list]) - for type, adapted_list in parametrized.items() - } - for typ, df in dataframes.items(): - - path_default = ( - TEST_DIR - / "_files" - / "tabular_datapackage_mininmal_example" - / "data" - / "elements" - / f"{typ}.csv" - ) - - df_default = pd.read_csv(path_default, sep=";") - - assert set(df.columns) == set(df_default.columns) - - pd.testing.assert_frame_equal(df, df_default) +# import pathlib +# from collections import namedtuple +# +# import pandas as pd +# +# from data_adapter_oemof.mappings import Mapper +# from data_adapter_oemof.adapters import VolatileAdapter, FACADE_ADAPTERS +# +# TEST_DIR = pathlib.Path(__file__).parent +# TEMP_DIR = TEST_DIR / "_temp" +# +# +# def test_simple_adapter(): +# data = { +# "type": "volatile", +# "region": "B", +# "carrier": "wind", +# "bus": "electricity", +# "tech": "onshore", +# "capacity": 12, +# "sedos-marginal_cost": 1, +# "sedos-overnight_cost": 1000, +# "capital_costs": 100, +# "sedos-lifetime": 25, +# "sedos-wacc": 0.05, +# } +# timeseries = pd.DataFrame({"a": [0, 0, 0], "onshore_B": [1, 2, 3]}) +# struct = {"default": {"inputs": ["onshore"], "outputs": ["electricity"]}} +# mapping = { +# "modex_tech_wind_turbine_onshore": {"profile": "onshore", "lifetime": "sedos-lifetime"} +# } +# mapper = Mapper( +# adapter=VolatileAdapter, +# process_name="modex_tech_wind_turbine_onshore", +# data=data, +# timeseries=timeseries, +# mapping=mapping, +# ) +# adapter = VolatileAdapter(struct, mapper) +# expected_dict = { +# "bus": "electricity", +# "capacity": 12, +# "carrier": "wind", +# "name": "B-None-onshore", +# "profile": "onshore_B", +# "tech": "onshore", +# "type": "volatile", +# "lifetime": 25, +# "region": "B", +# "year": None +# } +# assert adapter.as_dict() == expected_dict +# +# +# def test_adapter(): +# Component = namedtuple("Component", ["type", "data"]) +# +# # first, a list of components and some data +# components = [ +# Component( +# type="volatile", +# data={ +# "type": "volatile", +# "region": "B", +# "carrier": "wind", +# "tech": tech, +# "capacity": 12, +# "sedos-marginal_cost": 1, +# "sedos-overnight_cost": 1000, +# "capital_costs": 100, +# "sedos-lifetime": 25, +# "sedos-wacc": 0.05, +# }, +# ) +# for tech in ["onshore", "offshore"] +# ] +# +# struct = {"default": {"inputs": ["onshore"], "outputs": ["electricity"]}} +# +# # collect instances of the adapters +# parametrized = {} +# for component in components: +# if component.type not in parametrized: +# parametrized[component.type] = [] +# +# adapter = TYPE_MAP[component.type] +# +# parametrized[component.type].append( +# adapter.parametrize_dataclass(component.data, struct, None) +# ) +# # create a dictionary of dataframes +# dataframes = { +# type: pd.DataFrame([adapted.as_dict() for adapted in adapted_list]) +# for type, adapted_list in parametrized.items() +# } +# for typ, df in dataframes.items(): +# +# path_default = ( +# TEST_DIR +# / "_files" +# / "tabular_datapackage_mininmal_example" +# / "data" +# / "elements" +# / f"{typ}.csv" +# ) +# +# df_default = pd.read_csv(path_default, sep=";") +# +# assert set(df.columns) == set(df_default.columns) +# +# pd.testing.assert_frame_equal(df, df_default) From 408e419c20a7d1cf6181ca449fde35ff95c3c933 Mon Sep 17 00:00:00 2001 From: FelixMau Date: Mon, 2 Oct 2023 14:18:01 +0200 Subject: [PATCH 5/9] Dataclass dependencies To create a Mapper instance a adapter is neccessary but to create an adapter instance a mapper is neccessary as well. Therefore a new function has been created in Mapper to do the job the adapter instance did before (giving out all names and types of occouring fields) --- data_adapter_oemof/adapters.py | 7 +++++-- data_adapter_oemof/build_datapackage.py | 9 +++++---- data_adapter_oemof/mappings.py | 15 +++++++++++---- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/data_adapter_oemof/adapters.py b/data_adapter_oemof/adapters.py index 67e222f..a32370d 100644 --- a/data_adapter_oemof/adapters.py +++ b/data_adapter_oemof/adapters.py @@ -4,7 +4,7 @@ from oemof.tabular._facade import Facade from data_adapter_oemof import calculations -from data_adapter_oemof.mappings import Mapper +from data_adapter_oemof.mappings import Mapper, field_mock logger = logging.getLogger() @@ -12,7 +12,10 @@ class Adapter: type: str = "adapter" facade: Facade = None - extra_attributes = ("name", "type") + extra_attributes = ( + field_mock(name="name", type=str), + field_mock(name="type", type=str), + ) def as_dict(self) -> dict: return self.facade_dict diff --git a/data_adapter_oemof/build_datapackage.py b/data_adapter_oemof/build_datapackage.py index 6288b38..3536358 100644 --- a/data_adapter_oemof/build_datapackage.py +++ b/data_adapter_oemof/build_datapackage.py @@ -175,7 +175,7 @@ def get_foreign_keys(struct: list, mapper: Mapper, components: list) -> list: {"fields": bus, "reference": {"fields": "name", "resource": "bus"}} ) - for field in dataclasses.fields(mapper.adapter): + for field in mapper.fields_names(): if ( mapper.is_sequence(field.type) and field.name in components.columns @@ -393,10 +393,11 @@ def build_datapackage(cls, adapter: Adapter): data=component_data, timeseries=timeseries, ) - component = facade_adapter.parametrize_dataclass( - struct, component_mapper + components.append( + FACADE_ADAPTERS[facade_adapter_name]( + struct=struct, mapper=component_mapper + ).facade_dict ) - components.append(component.as_dict()) # Fill with all busses occurring, needed for foreign keys as well! process_busses += list(component_mapper.get_busses(struct).values()) diff --git a/data_adapter_oemof/mappings.py b/data_adapter_oemof/mappings.py index 87f436f..2f5a109 100644 --- a/data_adapter_oemof/mappings.py +++ b/data_adapter_oemof/mappings.py @@ -1,3 +1,4 @@ +import collections import dataclasses import difflib import logging @@ -15,6 +16,8 @@ "tech": "tech", } +field_mock = collections.namedtuple(typename="field_mock", field_names=["name", "type"]) + class MappingError(Exception): """Raised if mapping fails""" @@ -41,6 +44,12 @@ def __init__( self.mapping = mapping self.bus_map = bus_map + def fields_names(self): + return [ + field_mock(name=field.name, type=field.type) + for field in dataclasses.fields(self.adapter.facade) + ] + list(self.adapter.extra_attributes) + def map_key(self, key): """Use adapter specific mapping if available, otherwise use default mapping or return key if no mapping is available. @@ -143,9 +152,7 @@ def get_busses(self, struct): :return: dictionary with tabular like Busses """ bus_occurrences_in_fields = [ - field.name - for field in dataclasses.fields(self.adapter.facade) - if "bus" in field.name + field.name for field in self.fields_names() if "bus" in field.name ] if len(bus_occurrences_in_fields) == 0: logger.warning( @@ -213,7 +220,7 @@ def get_default_mappings(self, struct): mapped_all_class_fields = { field.name: value - for field in dataclasses.fields(self.adapter.facade) + for field in self.fields_names() if (value := self.get(field.name, field.type)) is not None } mapped_all_class_fields.update(self.get_busses(struct)) From 23e1c3cb53ecc73116fda91ff0414f960c8e9f72 Mon Sep 17 00:00:00 2001 From: Hendrik Huyskens Date: Fri, 6 Oct 2023 10:16:33 +0200 Subject: [PATCH 6/9] Fix test mapping with sequence --- tests/test_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_mapping.py b/tests/test_mapping.py index 75b6347..cbb1d97 100644 --- a/tests/test_mapping.py +++ b/tests/test_mapping.py @@ -74,7 +74,7 @@ def test_get_with_sequence(): "condensing_efficiency": "condensing_efficiency_DE", } - type = {i.name: i.type for i in dataclasses.fields(mapper.adapter)} + type = {i.name: i.type for i in dataclasses.fields(mapper.adapter.facade)} for key, _ in expected.items(): assert expected[key] == mapper.get(key, type.get(key)) From 49971f3462d9055e14d1fdd2f5956fdfa438977d Mon Sep 17 00:00:00 2001 From: Hendrik Huyskens Date: Fri, 6 Oct 2023 10:24:18 +0200 Subject: [PATCH 7/9] Minor naming change --- data_adapter_oemof/adapters.py | 8 ++++---- data_adapter_oemof/build_datapackage.py | 2 +- data_adapter_oemof/mappings.py | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/data_adapter_oemof/adapters.py b/data_adapter_oemof/adapters.py index a32370d..53005ab 100644 --- a/data_adapter_oemof/adapters.py +++ b/data_adapter_oemof/adapters.py @@ -4,7 +4,7 @@ from oemof.tabular._facade import Facade from data_adapter_oemof import calculations -from data_adapter_oemof.mappings import Mapper, field_mock +from data_adapter_oemof.mappings import Field, Mapper logger = logging.getLogger() @@ -12,9 +12,9 @@ class Adapter: type: str = "adapter" facade: Facade = None - extra_attributes = ( - field_mock(name="name", type=str), - field_mock(name="type", type=str), + extra_fields = ( + Field(name="name", type=str), + Field(name="type", type=str), ) def as_dict(self) -> dict: diff --git a/data_adapter_oemof/build_datapackage.py b/data_adapter_oemof/build_datapackage.py index 3536358..e9697e6 100644 --- a/data_adapter_oemof/build_datapackage.py +++ b/data_adapter_oemof/build_datapackage.py @@ -175,7 +175,7 @@ def get_foreign_keys(struct: list, mapper: Mapper, components: list) -> list: {"fields": bus, "reference": {"fields": "name", "resource": "bus"}} ) - for field in mapper.fields_names(): + for field in mapper.get_fields(): if ( mapper.is_sequence(field.type) and field.name in components.columns diff --git a/data_adapter_oemof/mappings.py b/data_adapter_oemof/mappings.py index 2f5a109..7716c5b 100644 --- a/data_adapter_oemof/mappings.py +++ b/data_adapter_oemof/mappings.py @@ -16,7 +16,7 @@ "tech": "tech", } -field_mock = collections.namedtuple(typename="field_mock", field_names=["name", "type"]) +Field = collections.namedtuple(typename="Field", field_names=["name", "type"]) class MappingError(Exception): @@ -44,11 +44,11 @@ def __init__( self.mapping = mapping self.bus_map = bus_map - def fields_names(self): + def get_fields(self): return [ - field_mock(name=field.name, type=field.type) + Field(name=field.name, type=field.type) for field in dataclasses.fields(self.adapter.facade) - ] + list(self.adapter.extra_attributes) + ] + list(self.adapter.extra_fields) def map_key(self, key): """Use adapter specific mapping if available, otherwise use default @@ -152,7 +152,7 @@ def get_busses(self, struct): :return: dictionary with tabular like Busses """ bus_occurrences_in_fields = [ - field.name for field in self.fields_names() if "bus" in field.name + field.name for field in self.get_fields() if "bus" in field.name ] if len(bus_occurrences_in_fields) == 0: logger.warning( @@ -220,7 +220,7 @@ def get_default_mappings(self, struct): mapped_all_class_fields = { field.name: value - for field in self.fields_names() + for field in self.get_fields() if (value := self.get(field.name, field.type)) is not None } mapped_all_class_fields.update(self.get_busses(struct)) From f28a45331ef59b0cca196c76721b8567d3a8ef7f Mon Sep 17 00:00:00 2001 From: Hendrik Huyskens Date: Fri, 6 Oct 2023 10:29:40 +0200 Subject: [PATCH 8/9] Minor change --- data_adapter_oemof/adapters.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/data_adapter_oemof/adapters.py b/data_adapter_oemof/adapters.py index 53005ab..00acfd0 100644 --- a/data_adapter_oemof/adapters.py +++ b/data_adapter_oemof/adapters.py @@ -17,9 +17,6 @@ class Adapter: Field(name="type", type=str), ) - def as_dict(self) -> dict: - return self.facade_dict - def __init__(self, struct: dict, mapper: Mapper): self.facade_dict = self.get_default_parameters(struct, mapper) From e539037b3b7b3875dfadf907baf8d46651c06cfb Mon Sep 17 00:00:00 2001 From: Hendrik Huyskens Date: Fri, 6 Oct 2023 11:17:21 +0200 Subject: [PATCH 9/9] Add region and year to extra fields and remove additional attributes --- data_adapter_oemof/adapters.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/data_adapter_oemof/adapters.py b/data_adapter_oemof/adapters.py index 00acfd0..8af9a84 100644 --- a/data_adapter_oemof/adapters.py +++ b/data_adapter_oemof/adapters.py @@ -15,24 +15,18 @@ class Adapter: extra_fields = ( Field(name="name", type=str), Field(name="type", type=str), + Field(name="region", type=str), + Field(name="year", type=int), ) def __init__(self, struct: dict, mapper: Mapper): self.facade_dict = self.get_default_parameters(struct, mapper) def get_default_parameters(self, struct: dict, mapper: Mapper) -> dict: - defaults = { - "type": self.type, - } + defaults = {"type": self.type} # Add mapped attributes mapped_values = mapper.get_default_mappings(struct) defaults.update(mapped_values) - # Add additional attributes - attributes = { - "region": mapper.get("region"), - "year": mapper.get("year"), - } - defaults.update(attributes) # add name if found in data, else use calculation for name: if (name := mapper.get("name")) is not None: @@ -53,6 +47,7 @@ class DispatchableAdapter(Adapter): """ Dispatchable Adapter """ + type = "dispatchable" facade = facades.Dispatchable @@ -61,6 +56,7 @@ class HeatPumpAdapter(Adapter): """ HeatPump Adapter """ + type = "heat_pump" facade = facades.HeatPump @@ -69,6 +65,7 @@ class LinkAdapter(Adapter): """ Link Adapter """ + type = "link" facade = facades.Link @@ -77,6 +74,7 @@ class ReservoirAdapter(Adapter): """ Reservoir Adapter """ + type = "reservoir" facade = facades.Reservoir @@ -85,6 +83,7 @@ class ExcessAdapter(Adapter): """ Excess Adapter """ + type = "excess" facade = facades.Excess @@ -93,6 +92,7 @@ class BackpressureTurbineAdapter(Adapter): """ BackpressureTurbine Adapter """ + type = "backpressure_turbine" facade = facades.BackpressureTurbine @@ -101,6 +101,7 @@ class CommodityAdapter(Adapter): """ CommodityAdapter """ + type = "commodity" facade = facades.Commodity @@ -110,6 +111,7 @@ class ConversionAdapter(Adapter): ConversionAdapter To use Conversion, map the inputs and outputs within the structure to avoid deduction failure. """ + type = "conversion" facade = facades.Conversion @@ -118,6 +120,7 @@ class LoadAdapter(Adapter): """ LoadAdapter """ + type = "load" facade = facades.Load @@ -126,6 +129,7 @@ class StorageAdapter(Adapter): """ StorageAdapter """ + type = "storage" facade = facades.Storage @@ -134,6 +138,7 @@ class ExtractionTurbineAdapter(Adapter): """ ExtractionTurbineAdapter """ + type = "extraction_turbine" facade = facades.ExtractionTurbine @@ -142,11 +147,14 @@ class VolatileAdapter(Adapter): """ VolatileAdapter """ + type = "volatile" facade = facades.Volatile # Create a dictionary of all adapter classes defined in this module FACADE_ADAPTERS = { - adapter.type: adapter for name, adapter in globals().items() if name.endswith("Adapter") + adapter.type: adapter + for name, adapter in globals().items() + if name.endswith("Adapter") }