Merge branch 'master' into osm-prebuilt-0.6

PyPSA · Nov 19, 2024 · 18e687d · 18e687d
2 parents bf4a195 + b5cbaca
commit 18e687d
Show file tree

Hide file tree

Showing 21 changed files with 854 additions and 736 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -12,13 +12,16 @@ on:
     branches:
     - master
   schedule:
-  - cron: "0 5 * * *"
+  - cron: "0 5 * * 1-6"
+  - cron: "0 5 * * 0"
 
 # Cancel any in-progress runs when a new run is triggered
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  BASE_ENV: envs/environment.yaml
 
 jobs:
   run-tests:
@@ -27,14 +30,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu, macos, windows]
-        include:
-        - os: ubuntu
-          env_file: envs/linux-pinned.yaml
-        - os: macos
-          env_file: envs/macos-pinned.yaml
-        - os: windows
-          env_file: envs/windows-pinned.yaml
+        # Run windows only on scheduled runs on Sundays, otherwise ignore
+        os: ${{ github.event.schedule == '0 5 * * 0' && fromJson('["ubuntu", "macos", "windows"]') || fromJson('["ubuntu", "macos"]') }}
 
     defaults:
       run:
@@ -43,17 +40,36 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Setup secrets & cache dates
+    - name: Setup env file path (ubuntu)
+      if: matrix.os == 'ubuntu'
       run: |
-        echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
-        echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
+        echo "env_file=envs/linux-pinned.yaml" >> $GITHUB_ENV
+
+    - name: Setup env file path (macos and windows)
+      if: matrix.os != 'ubuntu'
+      run: |
+        echo "env_file=envs/${{ matrix.os }}-pinned.yaml" >> $GITHUB_ENV
+
+    - name: Use base env file if it was changed
+      run: |
+        git fetch origin master
+        if git diff --name-only origin/master | grep '${{ env.BASE_ENV }}'; then
+          echo "Base env ${{ env.BASE_ENV }} changed. Using it instead of pinned envs."
+          echo "env_file=${{ env.BASE_ENV }}" >> $GITHUB_ENV
+        else
+          echo "Base env ${{ env.BASE_ENV }} not changed. Using pinned envs."
+        fi
+
+    - name: Setup cache keys
+      run: |
+        echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
 
     - uses: actions/cache@v4
       with:
         path: |
           data
           cutouts
-        key: data-cutouts-${{ env.week }}
+        key: data-cutouts-${{ env.WEEK }}
 
     - uses: conda-incubator/setup-miniconda@v3
       with:
@@ -63,12 +79,14 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ${{ env.CONDA }}/envs
-        key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(format('{0}', matrix.env_file)) }}
+        key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(format('{0}', env.env_file)) }}
       id: cache-env
 
     - name: Update environment
       if: steps.cache-env.outputs.cache-hit != 'true'
-      run: conda env update -n pypsa-eur -f ${{ matrix.env_file }}
+      run: |
+        conda env update -n pypsa-eur -f ${{ env.env_file }}
+        echo "Run conda list" && conda list
 
     - name: Run snakemake test workflows
       run: |
@@ -104,30 +122,39 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - name: Use base env file if it was changed
+      run: |
+        git fetch origin master
+        if git diff --name-only origin/master | grep '${{ env.BASE_ENV }}'; then
+          echo "Base env ${{ env.BASE_ENV }} changed. Using it instead of pinned envs."
+          echo "env_file=${{ env.BASE_ENV }}" >> $GITHUB_ENV
+        else
+          echo "Base env ${{ env.BASE_ENV }} not changed. Using pinned envs."
+          echo "env_file=envs/linux-pinned.yaml" >> $GITHUB_ENV
+        fi
+
     # Only run checks if package is not pinned
     - name: Check if inhouse package is pinned
       run: |
-        grep_line=$(grep -- '- ${{ matrix.inhouse }}' envs/environment.yaml)
+        grep_line=$(grep -- '- ${{ matrix.inhouse }}' ${{ env.BASE_ENV }})
         if [[ $grep_line == *"<"* || $grep_line == *"=="* ]]; then
             echo "pinned=true" >> $GITHUB_ENV
         else
             echo "pinned=false" >> $GITHUB_ENV
         fi
 
-    - name: Setup secrets & cache dates
+    - name: Setup cache keys
       if: env.pinned == 'false'
       run: |
-        echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
-        echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
-        echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env
+        echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
 
     - uses: actions/cache@v4
       if: env.pinned == 'false'
       with:
         path: |
           data
           cutouts
-        key: data-cutouts-${{ env.week }}
+        key: data-cutouts-${{ env.WEEK }}
 
     - uses: conda-incubator/setup-miniconda@v3
       if: env.pinned == 'false'
@@ -139,12 +166,14 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ${{ env.CONDA }}/envs
-        key: conda-${{ runner.os }}-${{ runner.arch }}-${{  matrix.inhouse  }}-hashFiles("envs/linux-pinned.yaml")
+        key: conda-${{ runner.os }}-${{ runner.arch }}-${{  matrix.inhouse  }}-${{ hashFiles(format('{0}', env.env_file)) }}
       id: cache-env
 
     - name: Update environment
       if: env.pinned == 'false' && steps.cache-env.outputs.cache-hit != 'true'
-      run: conda env update -n pypsa-eur -f envs/linux-pinned.yaml
+      run: |
+        conda env update -n pypsa-eur -f ${{ env.env_file }}
+        echo "Run conda list" && conda list
 
     - name: Install inhouse packages from master
       if: env.pinned == 'false'
@@ -157,7 +186,7 @@ jobs:
         make test
 
     - name: Upload artifacts
-      if: env.pinned == 'false'
+      if: env.pinned == 'false' && always()
       uses: actions/upload-artifact@v4
       with:
         name: results-inhouse-${{ matrix.inhouse }}

diff --git a/.github/workflows/update-fixed-env.yaml → .github/workflows/update-pinned-env.yaml b/.github/workflows/update-fixed-env.yaml → .github/workflows/update-pinned-env.yaml
@@ -2,17 +2,19 @@ name: Update pinned envs
 
 on:
   push:
-    branches:
-    - master
     paths:
-    - 'envs/environment.yaml'
+    - envs/environment.yaml
+  # Run every Sunday at 5:00 UTC
   schedule:
   - cron: "0 5 * * 0"
   workflow_dispatch:
 
+env:
+  BASE_ENV: envs/environment.yaml
 
 jobs:
   update-pinned-environment:
+    if: ${{ github.ref == 'refs/heads/master' }}
     name: Update pinned envs
     runs-on: ${{ matrix.os }}-latest
     strategy:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -89,6 +89,6 @@ repos:
 
   # Check for FSFE REUSE compliance (licensing)
 - repo: https://github.com/fsfe/reuse-tool
-  rev: v4.0.3
+  rev: v5.0.2
   hooks:
   - id: reuse
diff --git a/Makefile b/Makefile
@@ -54,12 +54,13 @@ clean-tests:
 
 # Removes all created files except for large cutout files (similar to fresh clone)
 reset:
-	@echo "Do you really wanna continue? This will remove logs, resources, benchmarks, results, and .snakemake directories (y/n): " && \
+	@echo "Do you really wanna continue? This will remove config/config.yaml, logs, resources, benchmarks, results, and .snakemake directories (y/n): " && \
 	read ans && [ $${ans} = y ] && ( \
 		rm -r ./logs || true; \
 		rm -r ./resources || true; \
 		rm -r ./benchmarks || true; \
 		rm -r ./results || true; \
 		rm -r ./.snakemake || true; \
+		rm ./config/config.yaml || true; \
 		echo "Reset completed." \
 	) || echo "Reset cancelled."
diff --git a/config/config.default.yaml b/config/config.default.yaml
@@ -499,7 +499,7 @@ sector:
   EV_lower_degree_factor: 0.98
   EV_upper_degree_factor: 0.63
   bev_dsm: true
-  bev_availability: 0.5
+  bev_dsm_availability: 0.5
   bev_energy: 0.05
   bev_charge_efficiency: 0.9
   bev_charge_rate: 0.011

diff --git a/doc/conf.py b/doc/conf.py
@@ -20,6 +20,22 @@
 import os
 import sys
 
+import requests
+
+
+def get_basemap(app):
+    url = "https://zenodo.org/records/14144752/files/map.html?download=1"
+    response = requests.get(url)
+
+    build_path = os.path.join(app.builder.outdir, "base-network-raw.html")
+    with open(build_path, "w") as f:
+        f.write(response.text)
+
+
+def setup(app):
+    app.connect("builder-inited", get_basemap)
+
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.

diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv
@@ -40,13 +40,13 @@ EV_lower_degree_factor,--,float,Share increase in energy demand in electric vehi
 EV_upper_degree_factor,--,float,Share increase in energy demand in electric vehicles (EV) for each degree difference between the hot environment and the maximum temperature.
 bev_dsm,--,"{true, false}",Add the option for battery electric vehicles (BEV) to participate in demand-side management (DSM)
 ,,,
-bev_availability,--,float,The share for battery electric vehicles (BEV) that are able to do demand side management (DSM)
+bev_dsm_availability,--,float,The share for battery electric vehicles (BEV) that are able to do demand side management (DSM)
 bev_energy,--,float,The average size of battery electric vehicles (BEV) in MWh
 bev_charge_efficiency,--,float,Battery electric vehicles (BEV) charge and discharge efficiency
 bev_charge_rate,MWh,float,The power consumption for one electric vehicle (EV) in MWh. Value derived from 3-phase charger with 11 kW.
 bev_avail_max,--,float,The maximum share plugged-in availability for passenger electric vehicles.
 bev_avail_mean,--,float,The average share plugged-in availability for passenger electric vehicles.
-v2g,--,"{true, false}",Allows feed-in to grid from EV battery
+v2g,--,"{true, false}","Allows feed-in to grid from EV battery. This is only enabled if BEV demand-side management is enabled, and the share of vehicles participating is V2G is given by `bev_dsm_availability`."
 land_transport_fuel_cell _share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses fuel cells in a given year
 land_transport_electric _share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses electric vehicles (EV) in a given year
 land_transport_ice _share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses internal combustion engines (ICE) in a given year. What is not EV or FCEV is oil-fuelled ICE.

diff --git a/doc/data-base-network.rst b/doc/data-base-network.rst
@@ -0,0 +1,45 @@
+..
+  SPDX-FileCopyrightText: 2024 The PyPSA-Eur Authors
+
+  SPDX-License-Identifier: CC-BY-4.0
+
+#############
+Base network
+#############
+
+.. raw:: html
+
+   <iframe src="base-network-raw.html" width="100%" height="600px"></iframe>
+
+The map might take a moment to load. To view it in full screen, click `here <base-network-raw.html>`__.
+
+``data/osm-prebuilt``
+
+- **Source:** OpenStreetMap; Xiong, B., Neumann, F., & Brown, T. (2024).
+  Prebuilt Electricity Network for PyPSA-Eur based on OpenStreetMap Data (0.5)
+  [Data set]. Zenodo. https://doi.org/10.5281/zenodo.13981528
+- **Link:** https://zenodo.org/records/13981528
+- **License:** ODbL (`reference <https://zenodo.org/records/13981528>`)
+- **Description:** Pre-built data of high-voltage transmission grid in Europe from OpenStreetMap.
+
+This dataset contains a topologically connected representation of the European
+high-voltage grid (220 kV to 750 kV) constructed using OpenStreetMap data. Input data
+was retrieved using the `Overpass turbo API <https://overpass-turbo.eu/>`__. A heurisitic
+cleaning process was used to for lines and links where electrical parameters are
+incomplete, missing, or ambiguous. Close substations within a radius of 500 m are
+aggregated to single buses, exact locations of underlying substations is preserved.
+Unique identifiers for lines and links are preserved.
+
+A detailed explanation on the background, methodology, and validation of this dataset
+can be found in `this paper <https://doi.org/10.48550/arXiv.2408.17178>`__ preprint
+currently under peer-review.
+
+Countries included in the dataset:
+
+Albania (AL), Austria (AT), Belgium (BE), Bosnia and Herzegovina (BA), Bulgaria (BG),
+Croatia (HR), Czech Republic (CZ), Denmark (DK), Estonia (EE), Finland (FI), France
+(FR), Germany (DE), Greece (GR), Hungary (HU), Ireland (IE), Italy (IT), Kosovo (XK),
+Latvia (LV), Lithuania (LT), Luxembourg (LU), Moldova (MD), Montenegro (ME), Netherlands
+(NL), North Macedonia (MK), Norway (NO), Poland (PL), Portugal (PT), Romania (RO), Serbia
+(RS), Slovakia (SK), Slovenia (SI), Spain (ES), Sweden (SE), Switzerland (CH), Ukraine
+(UA), United Kingdom (GB)