Merge remote-tracking branch 'origin/fix-and-extend-tests' into fix-a…

…nd-extend-tests

# Conflicts:
#	tests/_files/build_datapackage_goal/datapackage.json
#	tests/test_build_datapackage.py
#	tests/test_mapping.py

.github/workflows/linter.yml

@@ -0,0 +1,40 @@
+# This workflow will run linters with a single version of Python
+
+name: Black
+
+on:
+  push:
+    branches:
+      - main
+      - dev
+      - release/*
+  pull_request:
+
+jobs:
+  run-linters:
+    name: Run linters
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.9
+
+      - name: Install Python dependencies
+        run: pip install black flake8 isort
+
+      - name: Black
+        if: success() || failure()
+        run: black --check .
+
+      - name: flake8
+        if: success() || failure()
+        run: flake8 data_adapter_oemof tests
+
+      - name: isort
+        if: success() || failure()
+        run: isort --verbose --check-only --diff data_adapter_oemof tests

.github/workflows/pytest.yml

@@ -1,4 +1,4 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# This workflow will install Python dependencies, run tests with a single version of Python
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
 name: Pytest
@@ -11,8 +11,7 @@ on:
       - release/*
 
   pull_request:
-    branches:
-    - dev
+
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
@@ -24,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.8', '3.9', '3.10' ]
+        python-version: [ '3.9', '3.10' ]
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -60,10 +59,10 @@ jobs:
       - name: Test with pytest
         run: python -m poetry run python -m pytest -svvv
 
-      - name: Lint
-        run: |
-          python setup.py check --strict --metadata --restructuredtext
-          check-manifest .
-          black --check .
-          flake8 src tests
-          isort --verbose --check-only --diff src tests
+#      - name: Lint
+#        run: |
+#          python setup.py check --strict --metadata --restructuredtext
+#          check-manifest .
+#          black --check .
+#          flake8 src tests
+#          isort --verbose --check-only --diff src tests

.pre-commit-config.yaml

@@ -23,18 +23,13 @@ repos:
         entry: black
         language: python
         types: [ python ]
+      - id: isort
+        name: isort
+        entry: isort
+        language: python
+        types: [ python ]
       - id: flake8
         name: flake8
         entry: flake8
         language: python
         types: [python]
-      - id: pylint
-        name: pylint
-        entry: pylint
-        language: system
-        types: [python]
-        args:
-          [
-            "-rn", # Only display messages
-            "-sn", # Don't display the score
-          ]

conftest.py

@@ -0,0 +1,6 @@
+import pathlib
+
+import dotenv
+
+TEST_ENV_FILE = pathlib.Path(__file__).parent / "tests" / ".env"
+dotenv.load_dotenv(TEST_ENV_FILE)

data_adapter_oemof/adapters.py

@@ -1,16 +1,12 @@
 import dataclasses
 import logging
 
-import oemof.solph
-import pandas
-
-from oemof.tabular import facades
 from oemof.solph import Bus
+from oemof.tabular import facades
 
 from data_adapter_oemof import calculations
 from data_adapter_oemof.mappings import Mapper
 
-
 logger = logging.getLogger()
 
 

data_adapter_oemof/build_datapackage.py

@@ -1,13 +1,12 @@
 import dataclasses
+import os
 import warnings
 
 import pandas as pd
-import os
-
-from datapackage import Package
-
 from data_adapter import core
 from data_adapter.preprocessing import Adapter
+from datapackage import Package
+
 from data_adapter_oemof.adapters import FACADE_ADAPTERS
 from data_adapter_oemof.mappings import PROCESS_TYPE_MAP, Mapper
 
@@ -22,7 +21,8 @@ def refactor_timeseries(timeseries: pd.DataFrame):
     Parameters
     ----------
     timeseries: pd.DataFrame
-        timeseries in parameter-model format (https://github.com/sedos-project/oedatamodel#oedatamodel-parameter)
+        timeseries in parameter-model format
+        (https://github.com/sedos-project/oedatamodel#oedatamodel-parameter)
 
     Returns
     pd.DataFrame Tabular form of timeseries for multiple periods of similar
@@ -32,7 +32,6 @@ def refactor_timeseries(timeseries: pd.DataFrame):
     """
     # Combine all time series into one DataFrame
     df_timeseries = pd.DataFrame()
-    timeseries_timesteps = []
     if timeseries.empty:
         return df_timeseries
     # Iterate over different time periods/years
@@ -108,14 +107,16 @@ def get_foreign_keys(struct: list, mapper: Mapper, components: list) -> list:
         struct: list
             Energy System structure defining input/outputs for Processes
         mapper: Mapper
-            for one element of the Process (foreign keys have to be equal for all components of a Process)
+            for one element of the Process
+            (foreign keys have to be equal for all components of a Process)
         components: list
-            all components as of a Process as dicts. Helps to check what columns that could be pointing to sequences
-            are found in Sequences.
+            all components as of a Process as dicts. Helps to check what columns
+            that could be pointing to sequences are found in Sequences.
 
         Returns
         -------
-        list of foreignKeys for Process including bus references and pointers to files containing `profiles`
+        list of foreignKeys for Process including bus references and pointers to files
+        containing `profiles`
         """
         new_foreign_keys = []
         components = pd.DataFrame(components)
@@ -142,7 +143,8 @@ def get_foreign_keys(struct: list, mapper: Mapper, components: list) -> list:
                 elif any(components[field.name].isin(mapper.timeseries.columns)):
                     warnings.warn(
                         "Not all profile columns are set within the given profiles."
-                        f" Please check if there is a timeseries for every Component in {mapper.process_name}"
+                        f" Please check if there is a timeseries for every Component in "
+                        f"{mapper.process_name}"
                     )
                     new_foreign_keys.append(
                         {
@@ -153,7 +155,8 @@ def get_foreign_keys(struct: list, mapper: Mapper, components: list) -> list:
                         }
                     )
                 else:
-                    # Most likely the field may be a Timeseries in this case, but it is a scalar or unused.
+                    # Most likely the field may be a Timeseries in this case, but it
+                    # is a scalar or unused.
                     pass
         return new_foreign_keys
 
@@ -166,8 +169,9 @@ def save_datapackage_to_csv(self, destination: str) -> None:
         self: DataPackage
             DataPackage to save
         destination: str
-            String to where the datapackage save to. More convenient to use os.path. If last level of folder stucture
-            does not exist, it will be created (as well as /elements and /sequences)
+            String to where the datapackage save to. More convenient to use os.path.
+            If last level of folder stucture does not exist, it will be created
+            (as well as /elements and /sequences)
 
         Returns
         -------
@@ -229,8 +233,8 @@ def build_datapackage(cls, adapter: Adapter):
         Parameters
         ----------
         adapter: Adapter
-            Adapter from oemof_data_adapter that is able to handle parameter model data from Databus.
-            Adapter needs to be initialized with `structure_name`. Use `links_
+            Adapter from oemof_data_adapter that is able to handle parameter model data
+            from Databus. Adapter needs to be initialized with `structure_name`. Use `links_
 
         Returns
         -------
@@ -268,8 +272,8 @@ def build_datapackage(cls, adapter: Adapter):
             parametrized_elements["bus"] += process_busses
 
             # getting foreign keys with last component
-            #   foreign keys have to be equal for every component within a Process as foreign key columns cannot
-            #   have mixed meaning
+            #   foreign keys have to be equal for every component within a Process
+            #   as foreign key columns cannot have mixed meaning
             foreign_keys[process_name] = cls.get_foreign_keys(
                 struct, component_mapper, components
             )

data_adapter_oemof/mappings.py

@@ -135,9 +135,10 @@ def get_busses(self, struct):
             If found, search for similarities in the structure CSV.
             If not, search for name similarities:
                 Between the structure CSV and the adapter's buses take name from the structure.
-        Note: If passed class has more than two busses or different names for busses fields it is highly recommended
-        to provide BUS_NAME_MAP entry for this class. If multiple instances of the same facade
-        shall be having different inputs/outputs a facade Adapter has to be added for each.
+        Note: If passed class has more than two busses or different names for busses fields it
+        is highly recommended to provide BUS_NAME_MAP entry for this class.
+        If multiple instances of the same facade shall be having different inputs/outputs
+        a facade Adapter has to be added for each.
         :param struct: dict
         :return: dictionary with tabular like Busses
         """
@@ -153,7 +154,6 @@ def get_busses(self, struct):
 
         bus_dict = {}
         for bus in bus_occurrences_in_fields:  # emission_bus
-
             # 1. Check for existing mappings
             try:
                 bus_dict[bus] = self.bus_map[self.adapter.__name__][bus]
@@ -169,8 +169,8 @@ def get_busses(self, struct):
                 struct = struct["default"]
             else:
                 warnings.warn(
-                    "Please check structure and provide either one set of inputs/outputs or specify as default"
-                    "Parameter specific busses not implemented yet"
+                    "Please check structure and provide either one set of inputs/outputs "
+                    "or specify as default Parameter specific busses not implemented yet"
                 )
 
             # 2. Check for default busses

data_adapter_oemof/settings.py

@@ -1,4 +1,27 @@
+import os
 import pathlib
 
-ROOT_DIR = pathlib.Path(__file__).parent.parent
-COLLECTIONS_DIR = ROOT_DIR / "collections"
+ROOT_DIR = pathlib.Path(__file__).parent
+
+COLLECTIONS_DIR = (
+    pathlib.Path(os.environ["COLLECTIONS_DIR"])
+    if "COLLECTIONS_DIR" in os.environ
+    else pathlib.Path.cwd() / "collections"
+)
+if not COLLECTIONS_DIR.exists():
+    raise FileNotFoundError(
+        f"Could not find collections directory '{COLLECTIONS_DIR}'. "
+        "You should either create the collections folder or "
+        "change path to collection folder by changing environment variable 'COLLECTIONS_DIR'.",
+    )
+STRUCTURES_DIR = (
+    pathlib.Path(os.environ["STRUCTURES_DIR"])
+    if "STRUCTURES_DIR" in os.environ
+    else pathlib.Path.cwd() / "structures"
+)
+if not STRUCTURES_DIR.exists():
+    raise FileNotFoundError(
+        f"Could not find structure directory '{STRUCTURES_DIR}'. "
+        "You should either create the structure folder or "
+        "change path to structure folder by changing environment variable 'STRUCTURES_DIR'.",
+    )

docs/conf.py

@@ -12,6 +12,7 @@
 #
 import os
 import sys
+
 sys.path.insert(0, os.path.abspath('..'))