Merge pull request #9 from calliope-project/update-admin-border-workflow

Update administrative border workflow

Snakefile

@@ -66,7 +66,7 @@ rule test:
         "build/technically-eligible-land.tif",
         "build/technically-eligible-area-km2.tif",
         "build/technically-eligible-electricity-yield-pv-prio-twh.tif",
-        "build/administrative-borders-nuts.gpkg",
+        "build/administrative-borders.gpkg",
         "data/automatic/sonnendach/total-rooftop-area-km2.txt",
         "data/automatic/sonnendach/total-yield-twh.txt"
     output: "build/logs/test-report.html"

config/default.yaml

@@ -37,11 +37,19 @@ scope:
         - "Norway"
         - "Serbia"
         - "Switzerland"
+        - "Iceland"
     bounds:
-        x_min: -15.8 # in degrees east
+        x_min: -30 # in degrees east
         x_max: 37  # in degrees east
         y_min: 30  # in degrees north
         y_max: 75  # in degrees north
+    exclusion_zones:
+        atlantic_islands:
+            x_min: -30 # in degrees east
+            x_max: -10  # in degrees east
+            y_min: 30  # in degrees north
+            y_max: 41  # in degrees north
+
 layers:
     continental:
         Austria: nuts0
@@ -78,6 +86,7 @@ layers:
         Norway: nuts0
         Serbia: gadm0
         Switzerland: nuts0
+        Iceland: nuts0
     national:
         Austria: nuts0
         Belgium: nuts0
@@ -113,6 +122,7 @@ layers:
         Norway: nuts0
         Serbia: gadm0
         Switzerland: nuts0
+        Iceland: nuts0
     regional:
         # The link between NUTS and administrative units unfortunately is not obvious.
         # It's not documented anywhere -- at least I could not find any information.
@@ -152,6 +162,7 @@ layers:
         Norway: gadm1 # match 19
         Serbia: gadm1 # gadm1 25 regions; wiki 5 regions
         Switzerland: gadm1 # match 26
+        Iceland: gadm1
     municipal:
         Austria: lau2
         Belgium: lau2
@@ -187,6 +198,7 @@ layers:
         Norway: lau2
         Serbia: lau2
         Switzerland: lau2
+        Iceland: lau2
 parameters:
     maximum-installable-power-density: # this is not the yield, but the density of installed power
         pv-on-tilted-roofs: 160 # [W/m^2] from (Gagnon:2016, Klauser:2016), i.e. 16% efficiency

config/euler/cluster-config.yaml

@@ -18,8 +18,6 @@ elevation_in_europe:
     cores: 4
 eez_eligibility:
     cores: 4
-shared_coast:
-    cores: 4
 sensitivities:
     cores: 4
 capacityfactor_timeseries:
@@ -29,5 +27,5 @@ time_average_capacityfactor_map:
     memory: 32000
 national_boundaries:
     runtime: 60
-administrative_borders_lau:
-    runtime: 60
+administrative_borders:
+    runtime: 240

config/schema.yaml

@@ -7,5 +7,122 @@ properties:
             nuts-year:
                 type: number
                 enum: [2006, 2010, 2013, 2016, 2021]
-                default: 2016
-                description: Indicates the reference NUTS year
+                description: Indicates the reference NUTS year
+    crs:
+        type: string
+        enum: ["EPSG:4326"]
+        description: Coordinate reference system to which all datasets are reprojected during preprocessing
+    scope:
+        type: object
+        properties:
+            countries:
+                type: array
+                items:
+                    type: string
+                    enum:
+                        - "Austria"
+                        - "Belgium"
+                        - "Bulgaria"
+                        - "Croatia"
+                        - "Cyprus"
+                        - "Czech Republic"
+                        - "Denmark"
+                        - "Estonia"
+                        - "Finland"
+                        - "France"
+                        - "Germany"
+                        - "Greece"
+                        - "Hungary"
+                        - "Ireland"
+                        - "Italy"
+                        - "Latvia"
+                        - "Lithuania"
+                        - "Luxembourg"
+                        - "Netherlands"
+                        - "Poland"
+                        - "Portugal"
+                        - "Romania"
+                        - "Slovakia"
+                        - "Slovenia"
+                        - "Spain"
+                        - "Sweden"
+                        - "United Kingdom"
+                        - "Albania"
+                        - "Bosnia and Herzegovina"
+                        - "North Macedonia"
+                        - "Montenegro"
+                        - "Norway"
+                        - "Serbia"
+                        - "Switzerland"
+                        - "Iceland"
+                    description: Countries to include in the model
+            bounds:
+                type: object
+                properties:
+                    x_min:
+                        type: number
+                        minimum: -180
+                        maximum: 180
+                        description: Minimum longitude, in degrees east
+                    x_max:
+                        type: number
+                        minimum: -180
+                        maximum: 180
+                        description: Maximum longitude, in degrees east
+                    y_min:
+                        type: number
+                        minimum: -90
+                        maximum: 90
+                        description: Minimum Latitude, in degrees north
+                    y_max:
+                        type: number
+                        minimum: -90
+                        maximum: 90
+                        description: Maximum Latitude, in degrees north
+                description: Total extent of system under study. Defaults to all of Europe
+            exclusion_zones:
+                patternProperties:
+                    ^.*$:
+                        properties:
+                            x_min:
+                                type: number
+                                minimum: -180
+                                maximum: 180
+                                description: Minimum longitude, in degrees east
+                            x_max:
+                                type: number
+                                minimum: -180
+                                maximum: 180
+                                description: Maximum longitude, in degrees east
+                            y_min:
+                                type: number
+                                minimum: -90
+                                maximum: 90
+                                description: Minimum Latitude, in degrees north
+                            y_max:
+                                type: number
+                                minimum: -90
+                                maximum: 90
+                                description: Maximum Latitude, in degrees north
+                description: Any number of bounding boxes defining exclusion zones, where spatial features within the total bounds are to be ignored.
+    layers:
+        scope: object
+        patternProperties:
+            ^.*$:
+                type: object
+                properties:
+                    ^.*$:  # ideally this would be 'oneof' the list of countries above (can this be done in a schema??)
+                        type: string
+                        enum:
+                            - gadm0
+                            - gadm1
+                            - gadm2
+                            - gadm3
+                            - gadm4
+                            - gadm5
+                            - lau2
+                            - nuts0
+                            - nuts1
+                            - nuts2
+                            - nuts3
+

envs/default.yaml

@@ -13,7 +13,7 @@ dependencies:
   - fiona=1.8.4
   - rasterio=1.0.25
   - rasterstats=0.13.0
-  - geopandas=0.4.1
+  - geopandas=0.8.2
   - xarray=0.12.1
   - netcdf4=1.5.1.2
   - pyyaml=5.1

rules/data-preprocessing.smk

@@ -2,7 +2,7 @@
 import pycountry
 
 URL_LOAD = "https://data.open-power-system-data.org/time_series/2018-06-30/time_series_60min_stacked.csv"
-URL_NUTS = "https://ec.europa.eu/eurostat/cache/GISCO/distribution/v2/nuts/shp/NUTS_RG_01M_{}_4326.shp.zip"
+URL_NUTS = "https://ec.europa.eu/eurostat/cache/GISCO/distribution/v2/nuts/geojson/NUTS_RG_01M_{}_4326.geojson"
 URL_LAU = "http://ec.europa.eu/eurostat/cache/GISCO/geodatafiles/COMM-01M-2013-SH.zip"
 URL_DEGURBA = "http://ec.europa.eu/eurostat/cache/GISCO/geodatafiles/DGURBA_2014_SH.zip"
 URL_LAND_COVER = "http://due.esrin.esa.int/files/Globcover2009_V2.3_Global_.zip"
@@ -28,7 +28,7 @@ GMTED_Y = ["50N", "70N"]
 GMTED_X = ["030W", "000E", "030E"]
 
 localrules: raw_gadm_administrative_borders_zipped, raw_protected_areas_zipped,
-    raw_nuts_units_zipped, raw_lau_units_zipped, raw_land_cover_zipped,
+    raw_lau_units_zipped, raw_land_cover_zipped,
     raw_land_cover, raw_protected_areas, raw_srtm_elevation_tile_zipped, raw_gmted_elevation_tile,
     raw_bathymetry_zipped, raw_bathymetry, raw_gadm_administrative_borders
 
@@ -42,55 +42,33 @@ rule raw_gadm_administrative_borders_zipped:
 rule raw_gadm_administrative_borders:
     message: "Unzip administrative borders of {wildcards.country_code} as zip."
     input: "data/automatic/raw-gadm/{country_code}.zip"
-    output: temp("data/automatic/raw-gadm/gadm36_{country_code}.gpkg")
-    shell: "unzip -o {input} -d data/automatic/raw-gadm"
+    output: temp("build/raw-gadm/gadm36_{country_code}.gpkg")
+    shell: "unzip -o {input} -d build/raw-gadm"
 
 
-rule administrative_borders_gadm:
-    message: "Merge administrative borders of all countries up to layer {params.max_layer_depth}."
+rule all_gadm_administrative_borders:
+    message: "Merge gadm administrative borders of all countries."
     input:
-        "src/gadm.py",
-        ["data/automatic/raw-gadm/gadm36_{}.gpkg".format(country_code)
+        ["build/raw-gadm/gadm36_{}.gpkg".format(country_code)
             for country_code in [pycountry.countries.lookup(country).alpha_3
                                  for country in config['scope']['countries']]
          ]
-    params: max_layer_depth = 3
-    output: "build/administrative-borders-gadm.gpkg"
-    conda: "../envs/default.yaml"
-    shell:
-        PYTHON + " {input} {params.max_layer_depth} {output} {CONFIG_FILE}"
+    output: temp("build/raw-gadm/gadm36.gpkg")
+    params: crs = config["crs"]
+    conda: '../envs/default.yaml'
+    shell: "ogrmerge.py -o {output} -f gpkg -src_layer_field_content "{{LAYER_NAME}}" -t_srs {params.crs} -single {input}"
 
 
-rule raw_nuts_units_zipped:
-    message: "Download units as zip."
+rule raw_nuts_units:
+    message: "Download NUTS units as GeoJSON."
     output:
-        protected("data/automatic/raw-nuts{}-units.zip".format(config["parameters"]["nuts-year"]))
+        protected("data/automatic/raw-nuts{}-units.geojson".format(config["parameters"]["nuts-year"]))
     params:
         url = URL_NUTS.format(config["parameters"]["nuts-year"])
     shell:
         "curl -sLo {output} '{params.url}'"
 
 
-rule administrative_borders_nuts:
-    message: "Normalise NUTS administrative borders."
-    input:
-        src = "src/nuts.py",
-        zip = rules.raw_nuts_units_zipped.output
-    output:
-        "build/administrative-borders-nuts.gpkg"
-    shadow: "full"
-    params:
-        year = config['parameters']['nuts-year']
-    conda: "../envs/default.yaml"
-    shell:
-        """
-        unzip {input.zip} -d ./build/NUTS_RG_01M_{params.year}_4326/
-        {PYTHON} {input.src} to_multipolygon \
-        ./build/NUTS_RG_01M_{params.year}_4326 ./build/raw-nuts.gpkg
-        {PYTHON} {input.src} normalise ./build/raw-nuts.gpkg {output} {CONFIG_FILE}
-        """
-
-
 rule raw_lau_units_zipped:
     message: "Download LAU units as zip."
     output:
@@ -105,19 +83,31 @@ rule administrative_borders_lau:
         src = "src/lau.py",
         zip = rules.raw_lau_units_zipped.output
     output:
-        "build/administrative-borders-lau.geojson"
+        temp("build/raw-lau.gpkg")
     shadow: "full"
     conda: "../envs/default.yaml"
     shell:
         """
         unzip {input.zip} -d ./build
         {PYTHON} {input.src} merge ./build/COMM_01M_2013_SH/data/COMM_RG_01M_2013.shp \
         ./build/COMM_01M_2013_SH/data/COMM_AT_2013.dbf ./build/raw-lau.gpkg
-        {PYTHON} {input.src} identify ./build/raw-lau.gpkg ./build/raw-lau-identified.gpkg
-        {PYTHON} {input.src} normalise ./build/raw-lau-identified.gpkg {output} {CONFIG_FILE}
         """
 
 
+rule administrative_borders:
+    message: "Normalise all administrative borders."
+    input:
+        src = "src/administrative_borders.py",
+        nuts_geojson = rules.raw_nuts_units.output,
+        gadm_gpkg = rules.all_gadm_administrative_borders.output,
+        lau_gpkg = rules.administrative_borders_lau.output
+    output:
+        "build/administrative-borders.gpkg"
+    shadow: "full"
+    conda: "../envs/default.yaml"
+    shell: "{PYTHON} {input.src} {input.nuts_geojson} {input.gadm_gpkg} {input.lau_gpkg} {output} {CONFIG_FILE}"
+
+
 rule raw_land_cover_zipped:
     message: "Download land cover data as zip."
     output: protected("data/automatic/raw-globcover2009.zip")

rules/potential.smk

@@ -30,7 +30,7 @@ rule total_size_swiss_building_footprints_according_to_settlement_data:
         src = "src/swiss_building_footprints.py",
         building_footprints = rules.settlements.output.buildings,
         eligibility = "build/technically-eligible-land.tif",
-        countries = rules.administrative_borders_nuts.output[0]
+        countries = rules.administrative_borders.output
     output:
         "build/building-footprints-according-to-settlement-data-km2.txt"
     conda: "../envs/default.yaml"
@@ -124,9 +124,7 @@ rule units:
     message: "Form units of layer {wildcards.layer} by remixing NUTS, LAU, and GADM."
     input:
         "src/units.py",
-        rules.administrative_borders_nuts.output,
-        rules.administrative_borders_lau.output,
-        rules.administrative_borders_gadm.output
+        rules.administrative_borders.output,
     output:
         "build/{layer}/units.geojson"
     conda: "../envs/default.yaml"