Merge pull request #484 from PyPSA/merge-pypsa-eur-sec

Merge pypsa eur sec

.git-blame-ignore-revs

@@ -5,3 +5,4 @@
 # Exclude pre-commit applications
 5d1ef8a64055a039aa4a0834d2d26fe7752fe9a0
 92080b1cd2ca5f123158571481722767b99c2b27
+13769f90af4500948b0376d57df4cceaa13e78b5

.github/pull_request_template.md

@@ -7,7 +7,7 @@ Closes # (if applicable).
 
 - [ ] I tested my contribution locally and it seems to work fine.
 - [ ] Code and workflow changes are sufficiently documented.
-- [ ] Newly introduced dependencies are added to `envs/environment.yaml` and `envs/environment.docs.yaml`.
-- [ ] Changes in configuration options are added in all of `config.default.yaml`, `config.tutorial.yaml`, and `test/config.test1.yaml`.
-- [ ] Changes in configuration options are also documented in `doc/configtables/*.csv` and line references are adjusted in `doc/configuration.rst` and `doc/tutorial.rst`.
-- [ ] A note for the release notes `doc/release_notes.rst` is amended in the format of previous release notes.
+- [ ] Changed dependencies are added to `envs/environment.yaml`.
+- [ ] Changes in configuration options are added in all of `config.default.yaml`.
+- [ ] Changes in configuration options are also documented in `doc/configtables/*.csv`.
+- [ ] A release note `doc/release_notes.rst` is added.

.github/workflows/ci.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2021 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 
@@ -19,12 +19,14 @@ on:
   - cron: "0 5 * * TUE"
 
 env:
-  CACHE_NUMBER: 1 # Change this value to manually reset the environment cache
+  CONDA_CACHE_NUMBER: 1 # Change this value to manually reset the environment cache
+  DATA_CACHE_NUMBER: 2
 
 jobs:
   build:
 
     strategy:
+      fail-fast: false
       matrix:
         include:
           # Matrix required to handle caching with Mambaforge
@@ -49,16 +51,12 @@ jobs:
         shell: bash -l {0}
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Setup secrets
       run: |
         echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
 
-    - name: Add solver to environment
-      run: |
-        echo -e "- glpk\n- ipopt" >> envs/environment.yaml
-
     - name: Add solver to environment
       run: |
         echo -e "- glpk\n- ipopt<3.13.3" >> envs/environment.yaml
@@ -77,15 +75,25 @@ jobs:
         activate-environment: pypsa-eur
         use-mamba: true
 
-    - name: Set cache date
-      run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
+    - name: Set cache dates
+      run: |
+        echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
+        echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV
+
+    - name: Cache data and cutouts folders
+      uses: actions/cache@v3
+      with:
+        path: |
+          data
+          cutouts
+        key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }}
 
     - name: Create environment cache
       uses: actions/cache@v3
       id: cache
       with:
         path: ${{ matrix.prefix }}
-        key: ${{ matrix.label }}-conda-${{ hashFiles('envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
+        key: ${{ matrix.label }}-conda-${{ env.DATE }}-${{ env.CONDA_CACHE_NUMBER }}
 
     - name: Update environment due to outdated or unavailable cache
       run: mamba env update -n pypsa-eur -f envs/environment.yaml
@@ -95,4 +103,16 @@ jobs:
       run: |
         conda activate pypsa-eur
         conda list
-        snakemake -call solve_all_networks --configfile test/config.test1.yaml
+        snakemake -call solve_elec_networks --configfile test/config.electricity.yaml --rerun-triggers=mtime
+        snakemake -call all --configfile test/config.overnight.yaml --rerun-triggers=mtime
+        snakemake -call all --configfile test/config.myopic.yaml --rerun-triggers=mtime
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: resources-results
+        path: |
+          resources
+          results
+        if-no-files-found: warn
+        retention-days: 1

.gitignore

@@ -11,18 +11,65 @@ gurobi.log
 
 /bak
 /resources
+/resources*
 /results
 /networks
 /benchmarks
 /logs
 /notebooks
 /data
-/data/links_p_nom.csv
 /cutouts
-/dask-worker-space
 
 doc/_build
 
 config.yaml
 
 dconf
+/data/links_p_nom.csv
+/data/*totals.csv
+/data/biomass*
+/data/emobility/
+/data/eea*
+/data/jrc*
+/data/heating/
+/data/eurostat*
+/data/odyssee/
+/data/transport_data.csv
+/data/switzerland*
+/data/.nfs*
+/data/Industrial_Database.csv
+/data/retro/tabula-calculator-calcsetbuilding.csv
+/data/nuts*
+data/gas_network/scigrid-gas/
+data/costs_*.csv
+
+dask-worker-space/
+publications.jrc.ec.europa.eu/
+
+*.org
+
+*.nc
+
+*~
+/scripts/old
+
+*.pyc
+/cutouts
+/tmp
+/pypsa
+
+*.xlsx
+
+config.yaml
+
+doc/_build
+
+*.xls
+
+*.geojson
+
+*.ipynb
+
+data/costs_*
+
+merger-todos.md

.pre-commit-config.yaml

@@ -30,10 +30,10 @@ repos:
 
   # Find common spelling mistakes in comments and docstrings
 - repo: https://github.com/codespell-project/codespell
-  rev: v2.2.2
+  rev: v2.2.4
   hooks:
   - id: codespell
-    args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom'] # Ignore capital case words, e.g. country codes
+    args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore'] # Ignore capital case words, e.g. country codes
     types_or: [python, rst, markdown]
     files: ^(scripts|doc)/
 
@@ -74,7 +74,7 @@ repos:
 
   # Format Snakemake rule / workflow files
 - repo: https://github.com/snakemake/snakefmt
-  rev: v0.8.1
+  rev: v0.8.2
   hooks:
   - id: snakefmt
 
@@ -85,7 +85,7 @@ repos:
   - id: jupyter-notebook-cleanup
     exclude: examples/solve-on-remote.ipynb
 
-# Check for FSFE REUSE compliance (licensing)
+  # Check for FSFE REUSE compliance (licensing)
 - repo: https://github.com/fsfe/reuse-tool
   rev: v1.1.2
   hooks:

.reuse/dep5

@@ -1,28 +1,36 @@
 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: PyPSA-Eur
-Upstream-Contact: Tom Brown <tom.brown@kit.edu>
+Upstream-Contact: Tom Brown <t.brown@tu-berlin.de>
 Source: https://github.com/pypsa/pypsa-eur
 
 Files: doc/img/*
-Copyright: 2019 Fabian Neumann (TUB, KIT)
+Copyright: 2019-2023 The PyPSA-Eur Authors
+License: CC-BY-4.0
+
+Files: doc/data.csv
+Copyright: 2019-2023 The PyPSA-Eur Authors
 License: CC-BY-4.0
 
 Files: doc/configtables/*
-Copyright: 2019 Fabian Neumann (TUB, KIT)
+Copyright: 2019-2023 The PyPSA-Eur Authors
 License: CC-BY-4.0
 
 Files: data/*
 Copyright: 2017-2023 The PyPSA-Eur Authors
 License: CC-BY-4.0
 
 Files: .github/*
-Copyright: 2019 The PyPSA-Eur Authors
+Copyright: 2019-2023 The PyPSA-Eur Authors
 License: CC0-1.0
 
 Files: matplotlibrc
-Copyright: : 2017-2023 The PyPSA-Eur Authors
+Copyright: 2017-2023 The PyPSA-Eur Authors
 License: CC0-1.0
 
 Files: borg-it
-Copyright: : 2017-2023 The PyPSA-Eur Authors
+Copyright: 2017-2023 The PyPSA-Eur Authors
 License: CC0-1.0
+
+Files: graphics/*
+Copyright: 2017-2023 The PyPSA-Eur Authors
+License: CC-BY-4.0

.syncignore-receive

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2021 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 
@@ -15,5 +15,6 @@ __pycache__
 notebooks
 doc
 cutouts
-data/bundle
+data
+benchmarks
 *.nc

.syncignore-send

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2021 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 

README.md

@@ -7,41 +7,53 @@ SPDX-License-Identifier: CC-BY-4.0
 [![Build Status](https://github.com/pypsa/pypsa-eur/actions/workflows/ci.yaml/badge.svg)](https://github.com/PyPSA/pypsa-eur/actions)
 [![Documentation](https://readthedocs.org/projects/pypsa-eur/badge/?version=latest)](https://pypsa-eur.readthedocs.io/en/latest/?badge=latest)
 ![Size](https://img.shields.io/github/repo-size/pypsa/pypsa-eur)
-[![Zenodo](https://zenodo.org/badge/DOI/10.5281/zenodo.3520874.svg)](https://doi.org/10.5281/zenodo.3520874)
+[![Zenodo PyPSA-Eur](https://zenodo.org/badge/DOI/10.5281/zenodo.3520874.svg)](https://doi.org/10.5281/zenodo.3520874)
+[![Zenodo PyPSA-Eur-Sec](https://zenodo.org/badge/DOI/10.5281/zenodo.3938042.svg)](https://doi.org/10.5281/zenodo.3938042)
 [![Snakemake](https://img.shields.io/badge/snakemake-≥5.0.0-brightgreen.svg?style=flat)](https://snakemake.readthedocs.io)
 [![REUSE status](https://api.reuse.software/badge/github.com/pypsa/pypsa-eur)](https://api.reuse.software/info/github.com/pypsa/pypsa-eur)
 
-# PyPSA-Eur: An Open Optimisation Model of the European Transmission System
+# PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy System
 
-
-PyPSA-Eur is an open model dataset of the European power system at the
-transmission network level that covers the full ENTSO-E area.
-The model is suitable both for operational studies and generation and transmission expansion planning studies.
+PyPSA-Eur is an open model dataset of the European energy system at the
+transmission network level that covers the full ENTSO-E area. The model is suitable both for operational studies and generation and transmission expansion planning studies.
 The continental scope and highly resolved spatial scale enables a proper description of the long-range
 smoothing effects for renewable power generation and their varying resource availability.
 
+
+
+
 The model is described in the [documentation](https://pypsa-eur.readthedocs.io)
 and in the paper
 [PyPSA-Eur: An Open Optimisation Model of the European Transmission
 System](https://arxiv.org/abs/1806.01613), 2018,
 [arXiv:1806.01613](https://arxiv.org/abs/1806.01613).
+The model building routines are defined through a snakemake workflow.
+Please see the [documentation](https://pypsa-eur.readthedocs.io/)
+for installation instructions and other useful information about the snakemake workflow.
+The model is designed to be imported into the open toolbox
+[PyPSA](https://github.com/PyPSA/PyPSA).
 
-**WARNING**: Please read the [limitations](https://pypsa-eur.readthedocs.io/en/latest/limitations.html) section of the
-documentation and paper carefully before using the model. We do not
-recommend to use the full resolution network model for simulations. At
-high granularity the assignment of loads and generators to the nearest
-network node may not be a correct assumption, depending on the topology of the underlying distribution grid,
-and local grid
-bottlenecks may cause unrealistic load-shedding or generator
-curtailment. We recommend to cluster the network to a couple of
-hundred nodes to remove these local inconsistencies. See the
-discussion in Section 3.4 "Model validation" of the paper.
+**WARNING**: PyPSA-Eur is under active development and has several
+[limitations](https://pypsa-eur.readthedocs.io/en/latest/limitations.html)
+which you should understand before using the model. The github repository
+[issues](https://github.com/PyPSA/pypsa-eur/issues) collect known topics we are
+working on (please feel free to help or make suggestions). The
+[documentation](https://pypsa-eur.readthedocs.io/) remains somewhat patchy. You
+can find showcases of the model's capabilities in the preprint [Benefits of a
+Hydrogen Network in Europe](https://arxiv.org/abs/2207.05816), a [paper in Joule
+with a description of the industry sector](https://arxiv.org/abs/2109.09563), or
+in [a 2021 presentation at EMP-E](https://nworbmot.org/energy/brown-empe.pdf).
+We cannot support this model if you choose to use it. We do not recommend to use
+the full resolution network model for simulations. At high granularity the
+assignment of loads and generators to the nearest network node may not be a
+correct assumption, depending on the topology of the underlying distribution
+grid, and local grid bottlenecks may cause unrealistic load-shedding or
+generator curtailment. We recommend to cluster the network to a couple of
+hundred nodes to remove these local inconsistencies. See the discussion in
+Section 3.4 "Model validation" of the paper.
 
-![PyPSA-Eur Grid Model](doc/img/elec.png)
 
-The model building routines are defined through a snakemake workflow. The model is designed to be imported into the open toolbox
-[PyPSA](https://github.com/PyPSA/PyPSA) for operational studies as
-well as generation and transmission expansion planning studies.
+![PyPSA-Eur Grid Model](doc/img/elec.png)
 
 The dataset consists of:
 
@@ -57,9 +69,30 @@ The dataset consists of:
 - Renewable time series based on ERA5 and SARAH, assembled using the [atlite tool](https://github.com/FRESNA/atlite).
 - Geographical potentials for wind and solar generators based on land use (CORINE) and excluding nature reserves (Natura2000) are computed with the [atlite library](https://github.com/PyPSA/atlite).
 
+A sector-coupled extension adds demand
+and supply for the following sectors: transport, space and water
+heating, biomass, industry and industrial feedstocks, agriculture,
+forestry and fishing. This completes the energy system and includes
+all greenhouse gas emitters except waste management and land use.
+
+This diagram gives an overview of the sectors and the links between
+them:
+
+![sector diagram](graphics/multisector_figure.png)
+
+Each of these sectors is built up on the transmission network nodes
+from [PyPSA-Eur](https://github.com/PyPSA/pypsa-eur):
+
+![network diagram](https://github.com/PyPSA/pypsa-eur/blob/master/doc/img/base.png?raw=true)
+
+For computational reasons the model is usually clustered down
+to 50-200 nodes.
+
 Already-built versions of the model can be found in the accompanying [Zenodo
 repository](https://doi.org/10.5281/zenodo.3601881).
+# Licence
 
-A version of the model that adds building heating, transport and
-industry sectors to the model, as well as gas networks, can be found
-in the [PyPSA-Eur-Sec](https://github.com/PyPSA/pypsa-eur-sec) repository.
+The code in PyPSA-Eur is released as free software under the
+[MIT License](https://opensource.org/licenses/MIT), see `LICENSE.txt`.
+However, different licenses and terms of use may apply to the various
+input data.