Merge remote-tracking branch 'origin/master' into add-geothermal-sour…

…ced-heat-pumps

.github/workflows/test.yaml

@@ -12,13 +12,16 @@ on:
     branches:
     - master
   schedule:
-  - cron: "0 5 * * *"
+  - cron: "0 5 * * 1-6"
+  - cron: "0 5 * * 0"
 
 # Cancel any in-progress runs when a new run is triggered
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  BASE_ENV: envs/environment.yaml
 
 jobs:
   run-tests:
@@ -27,14 +30,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu, macos, windows]
-        include:
-        - os: ubuntu
-          env_file: envs/linux-pinned.yaml
-        - os: macos
-          env_file: envs/macos-pinned.yaml
-        - os: windows
-          env_file: envs/windows-pinned.yaml
+        # Run windows only on scheduled runs on Sundays, otherwise ignore
+        os: ${{ github.event.schedule == '0 5 * * 0' && fromJson('["ubuntu", "macos", "windows"]') || fromJson('["ubuntu", "macos"]') }}
 
     defaults:
       run:
@@ -43,17 +40,36 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Setup secrets & cache dates
+    - name: Setup env file path (ubuntu)
+      if: matrix.os == 'ubuntu'
       run: |
-        echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
-        echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
+        echo "env_file=envs/linux-pinned.yaml" >> $GITHUB_ENV
+
+    - name: Setup env file path (macos and windows)
+      if: matrix.os != 'ubuntu'
+      run: |
+        echo "env_file=envs/${{ matrix.os }}-pinned.yaml" >> $GITHUB_ENV
+
+    - name: Use base env file if it was changed
+      run: |
+        git fetch origin master
+        if git diff --name-only origin/master | grep '${{ env.BASE_ENV }}'; then
+          echo "Base env ${{ env.BASE_ENV }} changed. Using it instead of pinned envs."
+          echo "env_file=${{ env.BASE_ENV }}" >> $GITHUB_ENV
+        else
+          echo "Base env ${{ env.BASE_ENV }} not changed. Using pinned envs."
+        fi
+
+    - name: Setup cache keys
+      run: |
+        echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
 
     - uses: actions/cache@v4
       with:
         path: |
           data
           cutouts
-        key: data-cutouts-${{ env.week }}
+        key: data-cutouts-${{ env.WEEK }}
 
     - uses: conda-incubator/setup-miniconda@v3
       with:
@@ -63,12 +79,14 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ${{ env.CONDA }}/envs
-        key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(format('{0}', matrix.env_file)) }}
+        key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(format('{0}', env.env_file)) }}
       id: cache-env
 
     - name: Update environment
       if: steps.cache-env.outputs.cache-hit != 'true'
-      run: conda env update -n pypsa-eur -f ${{ matrix.env_file }}
+      run: |
+        conda env update -n pypsa-eur -f ${{ env.env_file }}
+        echo "Run conda list" && conda list
 
     - name: Run snakemake test workflows
       run: |
@@ -104,30 +122,39 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - name: Use base env file if it was changed
+      run: |
+        git fetch origin master
+        if git diff --name-only origin/master | grep '${{ env.BASE_ENV }}'; then
+          echo "Base env ${{ env.BASE_ENV }} changed. Using it instead of pinned envs."
+          echo "env_file=${{ env.BASE_ENV }}" >> $GITHUB_ENV
+        else
+          echo "Base env ${{ env.BASE_ENV }} not changed. Using pinned envs."
+          echo "env_file=envs/linux-pinned.yaml" >> $GITHUB_ENV
+        fi
+
     # Only run checks if package is not pinned
     - name: Check if inhouse package is pinned
       run: |
-        grep_line=$(grep -- '- ${{ matrix.inhouse }}' envs/environment.yaml)
+        grep_line=$(grep -- '- ${{ matrix.inhouse }}' ${{ env.BASE_ENV }})
         if [[ $grep_line == *"<"* || $grep_line == *"=="* ]]; then
             echo "pinned=true" >> $GITHUB_ENV
         else
             echo "pinned=false" >> $GITHUB_ENV
         fi
 
-    - name: Setup secrets & cache dates
+    - name: Setup cache keys
       if: env.pinned == 'false'
       run: |
-        echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
-        echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
-        echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env
+        echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
 
     - uses: actions/cache@v4
       if: env.pinned == 'false'
       with:
         path: |
           data
           cutouts
-        key: data-cutouts-${{ env.week }}
+        key: data-cutouts-${{ env.WEEK }}
 
     - uses: conda-incubator/setup-miniconda@v3
       if: env.pinned == 'false'
@@ -139,12 +166,14 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ${{ env.CONDA }}/envs
-        key: conda-${{ runner.os }}-${{ runner.arch }}-${{  matrix.inhouse  }}-hashFiles("envs/linux-pinned.yaml")
+        key: conda-${{ runner.os }}-${{ runner.arch }}-${{  matrix.inhouse  }}-${{ hashFiles(format('{0}', env.env_file)) }}
       id: cache-env
 
     - name: Update environment
       if: env.pinned == 'false' && steps.cache-env.outputs.cache-hit != 'true'
-      run: conda env update -n pypsa-eur -f envs/linux-pinned.yaml
+      run: |
+        conda env update -n pypsa-eur -f ${{ env.env_file }}
+        echo "Run conda list" && conda list
 
     - name: Install inhouse packages from master
       if: env.pinned == 'false'
@@ -157,7 +186,7 @@ jobs:
         make test
 
     - name: Upload artifacts
-      if: env.pinned == 'false'
+      if: env.pinned == 'false' && always()
       uses: actions/upload-artifact@v4
       with:
         name: results-inhouse-${{ matrix.inhouse }}

.github/workflows/update-fixed-env.yaml → .github/workflows/update-pinned-env.yaml

@@ -2,17 +2,19 @@ name: Update pinned envs
 
 on:
   push:
-    branches:
-    - master
     paths:
-    - 'envs/environment.yaml'
+    - envs/environment.yaml
+  # Run every Sunday at 5:00 UTC
   schedule:
   - cron: "0 5 * * 0"
   workflow_dispatch:
 
+env:
+  BASE_ENV: envs/environment.yaml
 
 jobs:
   update-pinned-environment:
+    if: ${{ github.ref == 'refs/heads/master' }}
     name: Update pinned envs
     runs-on: ${{ matrix.os }}-latest
     strategy:

.github/workflows/validate.yaml

@@ -16,7 +16,7 @@ jobs:
     if: github.event.pull_request.head.repo.full_name == github.repository
     runs-on: self-hosted
     steps:
-    - uses: lkstrp/[email protected].3
+    - uses: lkstrp/[email protected].4
       with:
         step: run-self-hosted-validation
         env_file: envs/environment.yaml
@@ -30,7 +30,7 @@ jobs:
     needs: run-validation
     runs-on: ubuntu-latest
     steps:
-    - uses: lkstrp/[email protected].3
+    - uses: lkstrp/[email protected].4
       with:
         step: create-comment
         snakemake_config: config/test/config.validator.yaml

.pre-commit-config.yaml

@@ -89,6 +89,6 @@ repos:
 
   # Check for FSFE REUSE compliance (licensing)
 - repo: https://github.com/fsfe/reuse-tool
-  rev: v4.0.3
+  rev: v5.0.2
   hooks:
   - id: reuse

Makefile

@@ -54,12 +54,13 @@ clean-tests:
 
 # Removes all created files except for large cutout files (similar to fresh clone)
 reset:
-	@echo "Do you really wanna continue? This will remove logs, resources, benchmarks, results, and .snakemake directories (y/n): " && \
+	@echo "Do you really wanna continue? This will remove config/config.yaml, logs, resources, benchmarks, results, and .snakemake directories (y/n): " && \
 	read ans && [ $${ans} = y ] && ( \
 		rm -r ./logs || true; \
 		rm -r ./resources || true; \
 		rm -r ./benchmarks || true; \
 		rm -r ./results || true; \
 		rm -r ./.snakemake || true; \
+		rm ./config/config.yaml || true; \
 		echo "Reset completed." \
 	) || echo "Reset cancelled."

Snakefile

@@ -138,3 +138,15 @@ rule sync:
         rsync -uvarh --no-g {params.cluster}/results . || echo "No results directory, skipping rsync"
         rsync -uvarh --no-g {params.cluster}/logs . || echo "No logs directory, skipping rsync"
         """
+
+
+rule sync_dry:
+    params:
+        cluster=f"{config['remote']['ssh']}:{config['remote']['path']}",
+    shell:
+        """
+        rsync -uvarh --ignore-missing-args --files-from=.sync-send . {params.cluster} -n
+        rsync -uvarh --no-g {params.cluster}/resources . -n || echo "No resources directory, skipping rsync"
+        rsync -uvarh --no-g {params.cluster}/results . -n || echo "No results directory, skipping rsync"
+        rsync -uvarh --no-g {params.cluster}/logs . -n || echo "No logs directory, skipping rsync"
+        """

config/config.default.yaml

@@ -84,9 +84,9 @@ co2_budget:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity
 electricity:
-  voltages: [200., 220., 300., 380., 500., 750.]
+  voltages: [220., 300., 330., 380., 400., 500., 750.]
   base_network: osm-prebuilt
-  osm-prebuilt-version: 0.4
+  osm-prebuilt-version: 0.5
   gaslimit_enable: false
   gaslimit: false
   co2limit_enable: false
@@ -278,10 +278,11 @@ conventional:
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#lines
 lines:
   types:
-    200.: "Al/St 240/40 2-bundle 200.0"
     220.: "Al/St 240/40 2-bundle 220.0"
     300.: "Al/St 240/40 3-bundle 300.0"
+    330.: "Al/St 240/40 3-bundle 300.0"
     380.: "Al/St 240/40 4-bundle 380.0"
+    400.: "Al/St 240/40 4-bundle 380.0"
     500.: "Al/St 240/40 4-bundle 380.0"
     750.: "Al/St 560/50 4-bundle 750.0"
   s_max_pu: 0.7
@@ -511,7 +512,7 @@ sector:
   EV_lower_degree_factor: 0.98
   EV_upper_degree_factor: 0.63
   bev_dsm: true
-  bev_availability: 0.5
+  bev_dsm_availability: 0.5
   bev_energy: 0.05
   bev_charge_efficiency: 0.9
   bev_charge_rate: 0.011
@@ -610,8 +611,12 @@ sector:
   overdimension_heat_generators:
     decentral: 1.1  #to cover demand peaks bigger than data
     central: 1.0
-  chp: true
-  micro_chp: false
+  chp:
+    enable: true
+    fuel:
+    - solid biomass # For solid biomass, CHP with and without CC are added
+    - gas # For all other fuels the same techno economic data from gas CHP is taken
+    micro_chp: false # Only gas is used for micro_chp
   solar_thermal: true
   solar_cf_correction: 0.788457  # =  >>> 1/1.2683
   marginal_cost_storage: 0. #1e-4

doc/conf.py

@@ -20,6 +20,22 @@
 import os
 import sys
 
+import requests
+
+
+def get_basemap(app):
+    url = "https://zenodo.org/records/14144752/files/map.html?download=1"
+    response = requests.get(url)
+
+    build_path = os.path.join(app.builder.outdir, "base-network-raw.html")
+    with open(build_path, "w") as f:
+        f.write(response.text)
+
+
+def setup(app):
+    app.connect("builder-inited", get_basemap)
+
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.

doc/configtables/electricity.csv

@@ -1,7 +1,7 @@
 ,Unit,Values,Description
-voltages,kV,"Any subset of {200., 220., 300., 380., 500., 750.}",Voltage levels to consider
+voltages,kV,"Any subset of {220., 300., 330., 380., 400., 500., 750.}",Voltage levels to consider
 base_network, --, "Any value in {'entsoegridkit', 'osm-prebuilt', 'osm-raw}", "Specify the underlying base network, i.e. GridKit (based on ENTSO-E web map extract, OpenStreetMap (OSM) prebuilt or raw (built from raw OSM data), takes longer."
-osm-prebuilt-version, --, "float, any value in range 0.1-0.4", "Choose the version of the prebuilt OSM network. Defaults to latest Zenodo release."
+osm-prebuilt-version, --, "float, any value in range 0.1-0.5", "Choose the version of the prebuilt OSM network. Defaults to latest Zenodo release."
 gaslimit_enable,bool,true or false,Add an overall absolute gas limit configured in ``electricity: gaslimit``.
 gaslimit,MWhth,float or false,Global gas usage limit
 co2limit_enable,bool,true or false,Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit`` in :mod:`prepare_network`. **Warning:** This option should currently only be used with electricity-only networks, not for sector-coupled networks..

doc/configtables/sector.csv

@@ -40,13 +40,13 @@ EV_lower_degree_factor,--,float,Share increase in energy demand in electric vehi
 EV_upper_degree_factor,--,float,Share increase in energy demand in electric vehicles (EV) for each degree difference between the hot environment and the maximum temperature.
 bev_dsm,--,"{true, false}",Add the option for battery electric vehicles (BEV) to participate in demand-side management (DSM)
 ,,,
-bev_availability,--,float,The share for battery electric vehicles (BEV) that are able to do demand side management (DSM)
+bev_dsm_availability,--,float,The share for battery electric vehicles (BEV) that are able to do demand side management (DSM)
 bev_energy,--,float,The average size of battery electric vehicles (BEV) in MWh
 bev_charge_efficiency,--,float,Battery electric vehicles (BEV) charge and discharge efficiency
 bev_charge_rate,MWh,float,The power consumption for one electric vehicle (EV) in MWh. Value derived from 3-phase charger with 11 kW.
 bev_avail_max,--,float,The maximum share plugged-in availability for passenger electric vehicles.
 bev_avail_mean,--,float,The average share plugged-in availability for passenger electric vehicles.
-v2g,--,"{true, false}",Allows feed-in to grid from EV battery
+v2g,--,"{true, false}","Allows feed-in to grid from EV battery. This is only enabled if BEV demand-side management is enabled, and the share of vehicles participating is V2G is given by `bev_dsm_availability`."
 land_transport_fuel_cell _share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses fuel cells in a given year
 land_transport_electric _share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses electric vehicles (EV) in a given year
 land_transport_ice _share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses internal combustion engines (ICE) in a given year. What is not EV or FCEV is oil-fuelled ICE.
@@ -93,8 +93,10 @@ biomass_boiler,--,"{true, false}",Add option for transforming biomass into heat
 overdimension_heat_generators,,,Add option for overdimensioning heating systems by a certain factor. This allows them to cover heat demand peaks e.g. 10% higher than those in the data with a setting of 1.1.
 -- decentral,--,float,The factor for overdimensioning (increasing CAPEX) decentral heating systems
 -- central,--,float,The factor for overdimensioning (increasing CAPEX) central heating systems
-chp,--,"{true, false}",Add option for using Combined Heat and Power (CHP)
-micro_chp,--,"{true, false}",Add option for using Combined Heat and Power (CHP) for decentral areas.
+chp,--,,
+-- enable,--,"{true, false}",Add option for using Combined Heat and Power (CHP)
+-- fuel,--,list of fuels,"Possible options are all fuels which have an existing bus and their CO2 intensity is given in the technology data. Currently possible are ""gas"", ""oil"", ""methanol"", ""lignite"", ""coal"" as well as ""solid biomass"". For all fuels except solid biomass, the techno-economic data from gas CHP is used. For the special case of solid biomass fuel, both CHP plants with and without carbon capture are added."
+-- micro_chp,--,"{true, false}",Add option for using gas-fired Combined Heat and Power (CHP) for decentral areas.
 solar_thermal,--,"{true, false}",Add option for using solar thermal to generate heat.
 solar_cf_correction,--,float,The correction factor for the value provided by the solar thermal profile calculations
 marginal_cost_storage,"currency/MWh ",float,The marginal cost of discharging batteries in distributed grids

doc/configuration.rst

@@ -150,7 +150,7 @@ Switches for some rules and optional features.
 
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
-   :start-at: enable:
+   :start-after: #enable
    :end-before: # docs
 
 .. csv-table::