Feature minimal workflow

README.md

@@ -107,6 +107,14 @@ Tests of models with continental and national resolution run automatically when
 
 Exchanging `regional` with `national` or `continental` allows you to run tests on the respective resolution explicitly.
 
+## Run minimal test
+
+As a developer, you may want to run the entire workflow often to spot errors early. For that, you can use a minimal test configuration that takes less time to run.
+
+    snakemake --use-conda --configfile="config/minimal.yaml"
+
+Make sure to run this in a clean working directory. Do not use the working directory in which you are using your normal configuration.
+
 ## License
 
 euro-calliope has been developed and is maintained by Tim Tröndle, IASS Potsdam. The code in this repository is MIT licensed.

Snakefile

@@ -91,14 +91,12 @@ rule parameterise_template:
 rule hydro_capacities:
     message: "Determine hydro capacities on {wildcards.resolution} resolution."
     input:
-        script = script_dir + "hydro.py",
+        script = script_dir + "hydro_capacities.py",
         locations = rules.units.output[0],
-        plants = rules.preprocess_hydro_stations.output[0],
-        phs_storage_capacities = config["data-sources"]["national-phs-storage-capacities"]
-    params: scale_phs = config["parameters"]["scale-phs-according-to-geth-et-al"]
+        plants = rules.preprocess_hydro_stations.output[0]
     output: "build/data/{resolution}/hydro-capacities-mw.csv"
     conda: "envs/geo.yaml"
-    script: "scripts/hydro.py"
+    script: "scripts/hydro_capacities.py"
 
 
 rule biofuels:

config/minimal.yaml

@@ -0,0 +1,28 @@
+# A simple config to allow for quick test runs of the workflow. It builds on `./default.yaml`.
+parameters:
+    scale-phs-according-to-geth-et-al: True # switched on to test the Geth scaling
+sea-connections:
+    continental:
+    national: # Source: https://www.entsoe.eu/data/map/
+        - [GBR, IRL]
+    regional: # Source: https://www.entsoe.eu/data/map/
+        - [GBR.4_1, IRL.17_1] # Wales and Meath
+scope:
+    countries:
+        - "Ireland"
+        - "United Kingdom"
+    bounds:
+        x_min: -10.62 # in degrees east
+        x_max: 1.8  # in degrees east
+        y_min: 49.8 # in degrees north
+        y_max: 60.9  # in degrees north
+shapes: # This config must be consistent with data from https://doi.org/10.5281/zenodo.3244985.
+    continental:
+        Ireland: nuts0
+        United Kingdom: nuts0
+    national:
+        Ireland: nuts0
+        United Kingdom: nuts0
+    regional:
+        Ireland: gadm1
+        United Kingdom: gadm1

envs/test.yaml

@@ -3,7 +3,7 @@ channels:
     - conda-forge
     - gurobi
 dependencies:
-    - python=3.8
+    - python=3.7
     - numpy=1.16.2
     - pandas=0.25.1
     - gurobi=8.1.1
@@ -12,3 +12,6 @@ dependencies:
     - calliope=0.6.5
     - pyomo=5.6.8 # to avoid https://github.com/calliope-project/calliope/issues/309
     - pyutilib=5.8.0 # older version that works with pyomo 5.6.8
+    - pip=21.0.1
+    - pip:
+        - -e ../../lib

rules/hydro.smk

@@ -59,26 +59,32 @@ rule stations_database:
         """
 
 
-rule fix_basins:
-    message: "Fix invalid basins."
+rule preprocess_basins:
+    message: "Preprocess basins."
     input:
-        script = script_dir + "hydro/fix_basins.py",
+        script = script_dir + "hydro/preprocess_basins.py",
         basins = rules.basins_database.output[0]
+    params:
+        x_min = config["scope"]["bounds"]["x_min"],
+        x_max = config["scope"]["bounds"]["x_max"],
+        y_min = config["scope"]["bounds"]["y_min"],
+        y_max = config["scope"]["bounds"]["y_max"]
     output: "build/data/hybas_eu_lev07_v1c.gpkg"
     conda: "../envs/hydro.yaml"
-    script: "../scripts/hydro/fix_basins.py"
+    script: "../scripts/hydro/preprocess_basins.py"
 
 
 rule preprocess_hydro_stations:
-    # Some locations of stations are imprecise and in the sea. Slightly move them.
-    # Some other stations seem incorrect. Remove.
-    # Add missing pumped hydro stations in Romania.
     message: "Preprocess hydro stations."
     input:
         script = script_dir + "hydro/preprocess_hydro_stations.py",
         stations = rules.stations_database.output[0],
-        basins = rules.fix_basins.output[0]
-    params: buffer_size = 1 / 60 # move stations up to 1 arcminute < 1 km
+        basins = rules.preprocess_basins.output[0],
+        phs_storage_capacities = config["data-sources"]["national-phs-storage-capacities"]
+    params:
+        buffer_size = 1 / 60, # move stations up to 1 arcminute < 1 km
+        countries = config["scope"]["countries"],
+        scale_phs = config["parameters"]["scale-phs-according-to-geth-et-al"]
     output: "build/data/jrc-hydro-power-plant-database-preprocessed.csv"
     conda: "../envs/hydro.yaml"
     script: "../scripts/hydro/preprocess_hydro_stations.py"
@@ -89,7 +95,7 @@ rule inflow_m3:
     input:
         script = script_dir + "hydro/inflow_m3.py",
         stations = rules.preprocess_hydro_stations.output[0],
-        basins = rules.fix_basins.output[0],
+        basins = rules.preprocess_basins.output[0],
         runoff = rules.download_runoff_data.output[0]
     params: year = config["year"]
     output: "build/data/hydro-electricity-with-water-inflow.nc"

scripts/hydro/fix_basins.py → scripts/hydro/preprocess_basins.py

@@ -3,26 +3,33 @@
 DRIVER = "GPKG"
 
 
-def fix_basins(path_to_basins, path_to_output):
-    """Fix the basins shapes which are invalid.
-
-    Following the advice given here:
-    https://github.com/Toblerity/Shapely/issues/344
-    """
-    basins = gpd.read_file(path_to_basins)
+def preprocess_basins(path_to_basins, bbox, path_to_output):
+    "Filter and fix basin shapes."
+    basins = gpd.read_file(path_to_basins, bbox=bbox)
     basins.geometry = basins.geometry.map(_buffer_if_necessary)
     basins.to_file(path_to_output, driver=DRIVER)
 
 
 def _buffer_if_necessary(shape):
+    """Fix the basins shapes which are invalid.
+
+    Following the advice given here:
+    https://github.com/Toblerity/Shapely/issues/344
+    """
     if not shape.is_valid:
         shape = shape.buffer(0.0)
     assert shape.is_valid
     return shape
 
 
 if __name__ == "__main__":
-    fix_basins(
+    preprocess_basins(
         path_to_basins=snakemake.input.basins,
+        bbox=(
+            snakemake.params.x_min,
+            snakemake.params.y_min,
+            snakemake.params.x_max,
+            snakemake.params.y_max
+        ),
         path_to_output=snakemake.output[0]
     )