From 810c25da5b86037366495652f4fdf90e8f4da959 Mon Sep 17 00:00:00 2001 From: Martha Frysztacki Date: Fri, 14 Jun 2024 09:09:25 +0200 Subject: [PATCH 1/8] fix sector coupeled model for single, non EU28 countries - e.g. UA (#1) * fix sector coupeled model for single, non EU28 countries - e.g. UA * Modification to industrial energy demand to use 0s if sector data unavailable --------- Co-authored-by: Sermisha --- doc/release_notes.rst | 3 ++ scripts/build_district_heat_share.py | 13 ++++--- scripts/build_energy_totals.py | 35 ++++++++++++++++++- ...ustrial_energy_demand_per_country_today.py | 15 ++++++-- ...industrial_energy_demand_per_node_today.py | 10 +++++- .../build_industrial_production_per_node.py | 9 +++++ 6 files changed, 76 insertions(+), 9 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index ae003d072..0b64372de 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -19,6 +19,9 @@ Upcoming Release `__. See configuration ``sector: enhanced_geothermal`` for details; by default switched off. +* Allow running the sector model for isolated non-EU28 countries, by filling missing sectoral + data with defaults, average EU values or zeros, if not available. + PyPSA-Eur 0.11.0 (25th May 2024) ===================================== diff --git a/scripts/build_district_heat_share.py b/scripts/build_district_heat_share.py index 178f2c0d9..30269cc92 100644 --- a/scripts/build_district_heat_share.py +++ b/scripts/build_district_heat_share.py @@ -33,9 +33,11 @@ pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) year = str(snakemake.params.energy_totals_year) - district_heat_share = pd.read_csv(snakemake.input.district_heat_share, index_col=0)[ - year - ] + district_heat_share = pd.read_csv(snakemake.input.district_heat_share, index_col=0) + if not district_heat_share.empty: + district_heat_share = district_heat_share[year] + else: + district_heat_share = pd.Series(index=pop_layout.index, data=0) # make ct-based share nodal district_heat_share = district_heat_share.reindex(pop_layout.ct).fillna(0) @@ -48,7 +50,8 @@ pop_layout["urban_ct_fraction"] = pop_layout.urban / pop_layout.ct.map(ct_urban.get) # fraction of node that is urban - urban_fraction = pop_layout.urban / pop_layout[["rural", "urban"]].sum(axis=1) + urban_fraction = (pop_layout.urban / pop_layout[["rural", "urban"]].sum(axis=1)).fillna(0) + # maximum potential of urban demand covered by district heating central_fraction = snakemake.config["sector"]["district_heating"]["potential"] @@ -56,7 +59,7 @@ # district heating share at each node dist_fraction_node = ( district_heat_share * pop_layout["urban_ct_fraction"] / pop_layout["fraction"] - ) + ).fillna(0) # if district heating share larger than urban fraction -> set urban # fraction to district heating share diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index fff95733a..2a343a23f 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -1026,7 +1026,40 @@ def update_residential_from_eurostat(energy): disable_progressbar=snakemake.config["run"].get("disable_progressbar", False), ) swiss = build_swiss() - idees = build_idees(idees_countries) + if len(idees_countries) > 0: + idees = build_idees(idees_countries) + else: + # e.g. UA and MD + logger.info(f"No IDEES data available for {countries} and years 2000-2015. Filling with zeros.") + years = range(2000, 2016) + idees = pd.DataFrame( + index=pd.MultiIndex.from_tuples([(country, year) for country in countries for year in years]), + columns=[ + "passenger cars", + "passenger car efficiency", + "total passenger cars", + "total other road passenger", + "total light duty road freight", + "total two-wheel", + "total heavy duty road freight", + "electricity passenger cars", + "electricity other road passenger", + "electricity light duty road freight", + "total rail passenger", + "total rail freight", + "electricity rail passenger", + "electricity rail freight", + "total domestic aviation passenger", + "total domestic aviation freight", + "total international aviation passenger", + "total international aviation freight", + "derived heat residential", + "derived heat services", + "thermal uses residential", + "thermal uses services", + ], + data=0 + ) energy = build_energy_totals(countries, eurostat, swiss, idees) diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 9c8f2e986..fb9ab0055 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -65,6 +65,7 @@ import country_converter as coco import pandas as pd +import numpy as np from _helpers import set_scenario_config from tqdm import tqdm @@ -195,12 +196,15 @@ def add_non_eu28_industrial_energy_demand(countries, demand, production): return demand eu28_production = production.loc[countries.intersection(eu28)].sum() + if eu28_production.sum() == 0: + logger.info("EU production is zero. Fallback: Filling non EU28 countries with zeros.") eu28_energy = demand.groupby(level=1).sum() eu28_averages = eu28_energy / eu28_production demand_non_eu28 = pd.concat( {k: v * eu28_averages for k, v in production.loc[non_eu28].iterrows()} - ) + ).fillna(0) + demand_non_eu28.replace([np.inf, -np.inf], 0, inplace=True) return pd.concat([demand, demand_non_eu28]) @@ -235,7 +239,14 @@ def industrial_energy_demand(countries, year): year = params.get("reference_year", 2015) countries = pd.Index(snakemake.params.countries) - demand = industrial_energy_demand(countries.intersection(eu28), year) + if len(countries.intersection(eu28)) > 0: + demand = industrial_energy_demand(countries.intersection(eu28), year) + else: + # e.g. only UA or MD + logger.info( + f"No industrial energy demand available for {countries}. Filling with average values of EU." + ) + demand = industrial_energy_demand(eu28, year) # output in MtMaterial/a production = ( diff --git a/scripts/build_industrial_energy_demand_per_node_today.py b/scripts/build_industrial_energy_demand_per_node_today.py index 7a1ee7ac5..75b4ea917 100644 --- a/scripts/build_industrial_energy_demand_per_node_today.py +++ b/scripts/build_industrial_energy_demand_per_node_today.py @@ -72,7 +72,15 @@ def build_nodal_industrial_energy_demand(): buses = keys.index[keys.country == country] mapping = sector_mapping.get(sector, "population") - key = keys.loc[buses, mapping] + try: + key = keys.loc[buses, mapping].fillna(0) + except: + logger.info( + f"No industrial demand available for {mapping}. Filling with zeros." + ) + keys[mapping] = 0 + key = keys.loc[buses, mapping].fillna(0) + demand = industrial_demand[country, sector] outer = pd.DataFrame( diff --git a/scripts/build_industrial_production_per_node.py b/scripts/build_industrial_production_per_node.py index d3edfa45b..d2cf13667 100644 --- a/scripts/build_industrial_production_per_node.py +++ b/scripts/build_industrial_production_per_node.py @@ -75,6 +75,15 @@ def build_nodal_industrial_production(): buses = keys.index[keys.country == country] mapping = sector_mapping.get(sector, "population") + try: + key = keys.loc[buses, mapping].fillna(0) + except: + logger.info( + f"No industrial production available for {mapping}. Filling with zeros." + ) + keys[mapping] = 0 + key = keys.loc[buses, mapping].fillna(0) + key = keys.loc[buses, mapping] nodal_production.loc[buses, sector] = ( industrial_production.at[country, sector] * key From f26fca69566730a883bf8066cae1068027288147 Mon Sep 17 00:00:00 2001 From: Martha Frysztacki Date: Wed, 19 Jun 2024 12:25:00 +0200 Subject: [PATCH 2/8] allow keeping existing capacities in sec from power (#4) * allow keeping existing capacities in sec from power * remove print statement * Fixes to keep caps (#6) * allow keeping existing capacities in sec from power * remove print statement * Fixes to prepare sector network to reflect existing conventional generations * Modifications to add renewable capacities from custom powerplants file * Update scripts/prepare_sector_network.py Co-authored-by: Martha Frysztacki * Update scripts/prepare_sector_network.py Co-authored-by: Martha Frysztacki * Fixing existing_efficiencies --------- Co-authored-by: martacki --------- Co-authored-by: SermishaNarayana <156903227+SermishaNarayana@users.noreply.github.com> --- config/config.default.yaml | 1 + doc/configtables/sector.csv | 1 + doc/release_notes.rst | 1 + rules/build_sector.smk | 1 + scripts/add_electricity.py | 11 +++++- scripts/prepare_sector_network.py | 62 +++++++++++++++++++++++++++---- 6 files changed, 69 insertions(+), 8 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 533c1e5dd..920fe2b1b 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -605,6 +605,7 @@ sector: biogas_upgrading_cc: false conventional_generation: OCGT: gas + keep_existing_capacities: false biomass_to_liquid: false biosng: false limit_max_growth: diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv index 059c42339..6857a86ff 100644 --- a/doc/configtables/sector.csv +++ b/doc/configtables/sector.csv @@ -136,6 +136,7 @@ biomass_spatial,--,"{true, false}",Add option for resolving biomass demand regio biomass_transport,--,"{true, false}",Add option for transporting solid biomass between nodes biogas_upgrading_cc,--,"{true, false}",Add option to capture CO2 from biomass upgrading conventional_generation,,,Add a more detailed description of conventional carriers. Any power generation requires the consumption of fuel from nodes representing that fuel. +keep_existing_capacities,--,"{true, false}",Keep existing conventional carriers from the power model. Defaults to false. biomass_to_liquid,--,"{true, false}",Add option for transforming solid biomass into liquid fuel with the same properties as oil biosng,--,"{true, false}",Add option for transforming solid biomass into synthesis gas with the same properties as natural gas limit_max_growth,,, diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 0b64372de..f6aa5fb7b 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -22,6 +22,7 @@ Upcoming Release * Allow running the sector model for isolated non-EU28 countries, by filling missing sectoral data with defaults, average EU values or zeros, if not available. +* Enable retaining exisiting conventional capacities added in the power only model for sector coupeled applications. PyPSA-Eur 0.11.0 (25th May 2024) ===================================== diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 6614b163a..f73e9672c 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -948,6 +948,7 @@ rule prepare_sector_network: countries=config_provider("countries"), adjustments=config_provider("adjustments", "sector"), emissions_scope=config_provider("energy", "emissions"), + electricity=config_provider("electricity"), RDIR=RDIR, input: unpack(input_profile_offwind), diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 49d0bdf75..b9099cad5 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -378,7 +378,7 @@ def update_transmission_costs(n, costs, length_factor=1.0): def attach_wind_and_solar( - n, costs, input_profiles, carriers, extendable_carriers, line_length_factor=1 + n, costs, ppl, input_profiles, carriers, extendable_carriers, line_length_factor=1 ): add_missing_carriers(n, carriers) for car in carriers: @@ -419,12 +419,20 @@ def attach_wind_and_solar( else: capital_cost = costs.at[car, "capital_cost"] + if not ppl.query("carrier == @car").empty: + caps = ppl.query("carrier == @car").groupby("bus").p_nom.sum() + caps = pd.Series(data = caps, index = ds.indexes["bus"]).fillna(0) + else: + caps = pd.Series(index = ds.indexes["bus"]).fillna(0) + n.madd( "Generator", ds.indexes["bus"], " " + car, bus=ds.indexes["bus"], carrier=car, + p_nom = caps, + p_nom_min = caps, p_nom_extendable=car in extendable_carriers["Generator"], p_nom_max=ds["p_nom_max"].to_pandas(), weight=ds["weight"].to_pandas(), @@ -863,6 +871,7 @@ def attach_line_rating( attach_wind_and_solar( n, costs, + ppl, snakemake.input, renewable_carriers, extendable_carriers, diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index ab4096e2c..351c2f49a 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -851,19 +851,17 @@ def annuity_factor(v): return costs -def add_generation(n, costs): +def add_generation(n, costs, existing_capacities=0, existing_efficiencies=None): logger.info("Adding electricity generation") nodes = pop_layout.index fallback = {"OCGT": "gas"} conventionals = options.get("conventional_generation", fallback) - for generator, carrier in conventionals.items(): carrier_nodes = vars(spatial)[carrier].nodes add_carrier_buses(n, carrier, carrier_nodes) - n.madd( "Link", nodes + " " + generator, @@ -874,9 +872,28 @@ def add_generation(n, costs): * costs.at[generator, "VOM"], # NB: VOM is per MWel capital_cost=costs.at[generator, "efficiency"] * costs.at[generator, "fixed"], # NB: fixed cost is per MWel - p_nom_extendable=True, + p_nom_extendable=( + True + if generator + in snakemake.params.electricity.get("extendable_carriers", dict()).get( + "Generator", list() + ) + else False + ), + p_nom=( + existing_capacities[generator] / existing_efficiencies[generator] + if not existing_capacities == 0 else 0 + ), # NB: existing capacities are MWel + p_max_pu = 0.7 if carrier == "uranium" else 1, # be conservative for nuclear (maintance or unplanned shut downs) + p_nom_min=( + existing_capacities[generator] if not existing_capacities == 0 else 0 + ), carrier=generator, - efficiency=costs.at[generator, "efficiency"], + efficiency=( + existing_efficiencies[generator] + if existing_efficiencies is not None + else costs.at[generator, "efficiency"] + ), efficiency2=costs.at[carrier, "CO2 intensity"], lifetime=costs.at[generator, "lifetime"], ) @@ -3934,6 +3951,28 @@ def add_enhanced_geothermal(n, egs_potentials, egs_overlap, costs): cyclic_state_of_charge=True, ) +def get_capacities_from_elec(n, carriers, component): + """ + Gets capacities and efficiencies for {carrier} in n.{component} that were + previously assigned in add_electricity. + """ + component_list = ["generators", "storage_units", "links", "stores"] + component_dict = {name: getattr(n, name) for name in component_list} + e_nom_carriers = ["stores"] + nom_col = {x: "e_nom" if x in e_nom_carriers else "p_nom" for x in component_list} + eff_col = "efficiency" + + capacity_dict = {} + efficiency_dict = {} + for carrier in carriers: + capacity_dict[carrier] = component_dict[component].query("carrier in @carrier")[ + nom_col[component] + ] + efficiency_dict[carrier] = component_dict[component].query( + "carrier in @carrier" + )[eff_col] + + return capacity_dict, efficiency_dict # %% if __name__ == "__main__": @@ -3980,11 +4019,20 @@ def add_enhanced_geothermal(n, egs_potentials, egs_overlap, costs): ) pop_weighted_energy_totals.update(pop_weighted_heat_totals) + if options.get("keep_existing_capacities", False): + existing_capacities, existing_efficiencies = get_capacities_from_elec( + n, + carriers=options.get("conventional_generation").keys(), + component="generators", + ) + else: + existing_capacities, existing_efficiencies = 0, None + patch_electricity_network(n) spatial = define_spatial(pop_layout.index, options) - if snakemake.params.foresight in ["myopic", "perfect"]: + if snakemake.params.foresight in ["overnight", "myopic", "perfect"]: add_lifetime_wind_solar(n, costs) conventional = snakemake.params.conventional_carriers @@ -3995,7 +4043,7 @@ def add_enhanced_geothermal(n, egs_potentials, egs_overlap, costs): add_co2_tracking(n, costs, options) - add_generation(n, costs) + add_generation(n, costs, existing_capacities, existing_efficiencies) add_storage_and_grids(n, costs) From c47ff3f3dc61d516291af85b47afeb2be4c7c37f Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 4 Oct 2024 20:47:59 +0200 Subject: [PATCH 3/8] Fix merge conflicts in prepare_sector_network --- scripts/prepare_sector_network.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 82fdc2833..430e8682e 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1111,13 +1111,8 @@ def add_generation(n, costs, existing_capacities=0, existing_efficiencies=None): nodes = pop_layout.index -<<<<<<< HEAD - fallback = {"OCGT": "gas"} - conventionals = options.get("conventional_generation", fallback) -======= conventionals = options["conventional_generation"] ->>>>>>> upstream/master for generator, carrier in conventionals.items(): carrier_nodes = vars(spatial)[carrier].nodes @@ -4615,7 +4610,6 @@ def get_capacities_from_elec(n, carriers, component): ) pop_weighted_energy_totals.update(pop_weighted_heat_totals) -<<<<<<< HEAD if options.get("keep_existing_capacities", False): existing_capacities, existing_efficiencies = get_capacities_from_elec( n, @@ -4625,8 +4619,6 @@ def get_capacities_from_elec(n, carriers, component): else: existing_capacities, existing_efficiencies = 0, None - patch_electricity_network(n) -======= landfall_lengths = { tech: settings["landfall_length"] for tech, settings in snakemake.params.renewable.items() @@ -4637,7 +4629,6 @@ def get_capacities_from_elec(n, carriers, component): fn = snakemake.input.heating_efficiencies year = int(snakemake.params["energy_totals_year"]) heating_efficiencies = pd.read_csv(fn, index_col=[1, 0]).loc[year] ->>>>>>> upstream/master spatial = define_spatial(pop_layout.index, options) From 61ee4b12f8eadf6a33c4beb8dd5f2ada2d86180b Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 10 Oct 2024 15:15:02 +0200 Subject: [PATCH 4/8] Update scripts/prepare_sector_network.py Co-authored-by: Martha Frysztacki --- scripts/prepare_sector_network.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 430e8682e..47ef6c6aa 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1111,7 +1111,8 @@ def add_generation(n, costs, existing_capacities=0, existing_efficiencies=None): nodes = pop_layout.index - conventionals = options["conventional_generation"] + fallback = {"OCGT": "gas"} + conventionals = options.get("conventional_generation", fallback) for generator, carrier in conventionals.items(): carrier_nodes = vars(spatial)[carrier].nodes From 2404ffa6df3a25742349d0cffb56bc0c5c93de65 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 11 Oct 2024 10:12:33 +0200 Subject: [PATCH 5/8] Fix indentation --- scripts/prepare_sector_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 47ef6c6aa..af939c2ef 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1111,8 +1111,8 @@ def add_generation(n, costs, existing_capacities=0, existing_efficiencies=None): nodes = pop_layout.index - fallback = {"OCGT": "gas"} - conventionals = options.get("conventional_generation", fallback) + fallback = {"OCGT": "gas"} + conventionals = options.get("conventional_generation", fallback) for generator, carrier in conventionals.items(): carrier_nodes = vars(spatial)[carrier].nodes From 89a9d071df57b86a16579528de4c1f2079b50ce8 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 12 Nov 2024 14:55:55 +0100 Subject: [PATCH 6/8] Add a reproducible workflow strategy compatible with soft-fork (#19) * Add OET flavor to repository to enable reproducible projects * Remove cookiecutter references to clean the code * Remove plots directory as graphics should be in results * Remove raw environment * Add the report in the all rule * Improve licensing scheme * Add AGPL-3.0-or-latex license * Remove reference to submodule * Add reference to micromamba and conda * Update README.md Co-authored-by: Siddharth Krishna * Update README.md Co-authored-by: Siddharth Krishna * Update README.md * Add reference to soft-fork strategy * Switch default license from AGPL to MIT --------- Co-authored-by: Siddharth Krishna --- .gitignore | 1 + .reuse/dep5 | 4 + README.md | 58 ++++++++- Snakefile | 2 + report/references.bib | 8 ++ report/report.tex | 205 ++++++++++++++++++++++++++++++ report/sections/.gitkeep | 3 + report/sections/abstract.tex | 5 + report/sections/conclusion.tex | 5 + report/sections/discussion.tex | 5 + report/sections/introduction.tex | 5 + report/sections/methods.tex | 6 + report/sections/results.tex | 5 + report/sections/supplementary.tex | 5 + report/static/.gitkeep | 3 + rules/report.smk | 19 +++ 16 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 report/references.bib create mode 100644 report/report.tex create mode 100644 report/sections/.gitkeep create mode 100644 report/sections/abstract.tex create mode 100644 report/sections/conclusion.tex create mode 100644 report/sections/discussion.tex create mode 100644 report/sections/introduction.tex create mode 100644 report/sections/methods.tex create mode 100644 report/sections/results.tex create mode 100644 report/sections/supplementary.tex create mode 100644 report/static/.gitkeep create mode 100755 rules/report.smk diff --git a/.gitignore b/.gitignore index 27a2fb182..680f7be6e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ __pycache__ *dconf gurobi.log .vscode +.idea *.orig /bak diff --git a/.reuse/dep5 b/.reuse/dep5 index a40d090a9..387fc9913 100644 --- a/.reuse/dep5 +++ b/.reuse/dep5 @@ -30,3 +30,7 @@ License: CC0-1.0 Files: borg-it Copyright: 2017-2024 The PyPSA-Eur Authors License: CC0-1.0 + +Files: report/* +Copyright: Open Energy Transition gGmbH +License: MIT diff --git a/README.md b/README.md index 1ed6f43cf..7ca589ebd 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,62 @@ SPDX-License-Identifier: CC-BY-4.0 [![REUSE status](https://api.reuse.software/badge/github.com/pypsa/pypsa-eur)](https://api.reuse.software/info/github.com/pypsa/pypsa-eur) [![Stack Exchange questions](https://img.shields.io/stackexchange/stackoverflow/t/pypsa)](https://stackoverflow.com/questions/tagged/pypsa) +# {{project_name}} +Open Energy Transition Logo + +This repository is a soft-fork of [PyPSA-Eur](https://github.com/pypsa/pypsa-eur) and contains the entire project `{{project_name}}` supported by [Open Energy Transition (OET)](https://openenergytransition.org/)*, including code and report. The philosophy behind this repository is that no intermediary results are included, but all results are computed from raw data and code. The structure is also inspired by [cookiecutter-project](https://github.com/PyPSA/cookiecutter-project). + +This repository is maintained using [OET's soft-fork strategy](https://open-energy-transition.github.io/handbook/docs/Engineering/SoftForkStrategy). OET's primary aim is to contribute as much as possible to the open source (OS) upstream repositories. For long-term changes that cannot be directly merged upstream, the strategy organizes and maintains OET forks, ensuring they remain up-to-date and compatible with upstream, while also supporting future contributions back to the OS repositories. + + +# Repository structure + +* `benchmarks`: will store `snakemake` benchmarks (does not exist initially) +* `config`: configurations used in the study +* `cutouts`: will store raw weather data cutouts from `atlite` (does not exist initially) +* `data`: includes input data that is not produced by any `snakemake` rule +* `doc`: includes all files necessary to build the `readthedocs` documentation of PyPSA-Eur +* `envs`: includes all the `mamba` environment specifications to run the workflow +* `logs`: will store log files (does not exist initially) +* `notebooks`: includes all the `notebooks` used for ad-hoc analysis +* `report`: contains all files necessary to build the report; plots and result files are generated automatically +* `rules`: includes all the `snakemake`rules loaded in the `Snakefile` +* `resources`: will store intermediate results of the workflow which can be picked up again by subsequent rules (does not exist initially) +* `results`: will store the solved PyPSA network data, summary files and plots (does not exist initially) +* `scripts`: includes all the Python scripts executed by the `snakemake` rules to build the model + +# Installation and Usage + +## 1. Installation + +Clone the repository: + + git clone https://github.com/open-energy-transition/{{repository}} + +You need [mamba](https://mamba.readthedocs.io/en/latest/) to run the analysis. Users may also prefer to use [micromamba](https://mamba.readthedocs.io/en/latest/installation/micromamba-installation.html) or [conda](https://docs.conda.io/projects/conda/en/latest/index.html). Using `mamba`, you can create an environment from within you can run it: + + mamba env create -f environment.yaml + +Activate the newly created `{{project_short_name}}` environment: + + mamba activate {{project_short_name}} + +## 2. Run the analysis + + snakemake -call + +This will run all analysis steps to reproduce results and build the report. + +To generate a PDF of the dependency graph of all steps `resources/dag.pdf` run: + + snakemake -c1 dag + +* Open Energy Transition (g)GmbH, Königsallee 52, 95448 Bayreuth, Germany + +---- + +---- + # PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy System PyPSA-Eur is an open model dataset of the European energy system at the @@ -105,4 +161,4 @@ We strongly welcome anyone interested in contributing to this project. If you ha The code in PyPSA-Eur is released as free software under the [MIT License](https://opensource.org/licenses/MIT), see [`doc/licenses.rst`](doc/licenses.rst). However, different licenses and terms of use may apply to the various -input data, see [`doc/data_sources.rst`](doc/data_sources.rst). +input data, see [`doc/data_sources.rst`](doc/data_sources.rst). \ No newline at end of file diff --git a/Snakefile b/Snakefile index 90a44c9d9..87566639c 100644 --- a/Snakefile +++ b/Snakefile @@ -54,6 +54,7 @@ include: "rules/solve_electricity.smk" include: "rules/postprocess.smk" include: "rules/validate.smk" include: "rules/development.smk" +include: "rules/report.smk" if config["foresight"] == "overnight": @@ -74,6 +75,7 @@ if config["foresight"] == "perfect": rule all: input: expand(RESULTS + "graphs/costs.svg", run=config["run"]["name"]), + "report/report.pdf" default_target: True diff --git a/report/references.bib b/report/references.bib new file mode 100644 index 000000000..c239cf057 --- /dev/null +++ b/report/references.bib @@ -0,0 +1,8 @@ +% SPDX-FileCopyrightText: Open Energy Transition gGmbH +% +% SPDX-License-Identifier: MIT + +@misc{ Nobody06, + author = "Nobody Jr", + title = "My Article", + year = "2006" } \ No newline at end of file diff --git a/report/report.tex b/report/report.tex new file mode 100644 index 000000000..7988756f6 --- /dev/null +++ b/report/report.tex @@ -0,0 +1,205 @@ +% SPDX-FileCopyrightText: Open Energy Transition gGmbH +% +% SPDX-License-Identifier: MIT + +\documentclass[5p]{elsarticle} + +\journal{TBA} + +\bibliographystyle{elsarticle-num} +\biboptions{numbers,sort&compress} + +% format hacks +\usepackage{libertine} +\usepackage{libertinust1math} +% \usepackage{geometry} +% \geometry{ +% top=30mm, +% bottom=35mm, +% } + +\usepackage{amsmath} +\usepackage{bbold} +\usepackage{graphicx} +\usepackage{eurosym} +\usepackage{mathtools} +\usepackage{url} +\usepackage{booktabs} +\usepackage{epstopdf} +\usepackage{xfrac} +\usepackage{tabularx} +\usepackage{bm} +\usepackage{subcaption} +\usepackage{blindtext} +\usepackage{longtable} +\usepackage{multirow} +\usepackage{threeparttable} +\usepackage{pdflscape} +\usepackage[export]{adjustbox} +\usepackage[version=4]{mhchem} +\usepackage[colorlinks]{hyperref} +\usepackage[parfill]{parskip} +\usepackage[nameinlink,sort&compress,capitalise,noabbrev]{cleveref} +\usepackage[leftcaption,raggedright]{sidecap} +\usepackage[prependcaption,textsize=footnotesize]{todonotes} + +\usepackage{siunitx} +\sisetup{ + range-units = single, + per-mode = symbol +} +\DeclareSIUnit\year{a} +\DeclareSIUnit{\tco}{t_{\ce{CO2}}} +\DeclareSIUnit{\sieuro}{\mbox{\euro}} + +\usepackage{lipsum} + +\usepackage[resetlabels,labeled]{multibib} +\newcites{S}{Supplementary References} +\bibliographystyleS{elsarticle-num} + +\graphicspath{ + {../results/graphics/}, +} + +% \usepackage[ +% type={CC}, +% modifier={by}, +% version={4.0}, +% ]{doclicense} + +\newcommand{\abs}[1]{\left|#1\right|} +\newcommand{\norm}[1]{\left\lVert#1\right\rVert} +\newcommand{\set}[1]{\left\{#1\right\} } +\DeclareMathOperator*{\argmin}{\arg\!\min} +\newcommand{\R}{\mathbb{R} } +\newcommand{\B}{\mathbb{B} } +\newcommand{\N}{\mathbb{N} } +\newcommand{\co}{\ce{CO2}~} +\def\el{${}_{\textrm{el}}$} +\def\th{${}_{\textrm{th}}$} +\def\deg{${}^\circ$} + +\begin{document} + + \begin{frontmatter} + + \title{ project name } + + \author[affil]{ name surname } % \corref{correspondingauthor} + \ead{ name.surname@openenergytransition.org } + + \address[affil]{ affiliation } + + \begin{abstract} + \input{sections/abstract.tex} + \end{abstract} + + \begin{keyword} + TODO + \end{keyword} + + % \begin{graphicalabstract} + % \end{graphicalabstract} + + \begin{highlights} + \item A + \item B + \item C + \end{highlights} + +\end{frontmatter} + +% \listoftodos[TODOs] + +% \tableofcontents + +\section{Introduction} +\label{sec:intro} + +\input{sections/introduction.tex} + +\section{Methods} +\label{sec:methods} + +\input{sections/methods.tex} + +\section{Results} +\label{sec:results} + +\input{sections/results.tex} + +\section{Discussion} +\label{sec:discussion} + +\input{sections/discussion.tex} + +\section{Conclusion} +\label{sec:conclusion} + +\input{sections/conclusion.tex} + +\section*{Acknowledgements} + +% XY gratefully acknowledge funding from XY under grant no. 00000. + +\section*{License} +% \doclicenseLongText +% \doclicenseIcon + +\section*{Author Contributions} + +% following https://casrai.org/credit/ + +\textbf{Author XY}: +Conceptualization -- +Data curation -- +Formal Analysis -- +Funding acquisition -- +Investigation -- +Methodology -- +Project administration -- +Resources -- +Software -- +Supervision -- +Validation -- +Visualization -- +Writing - original draft -- +Writing - review \& editing + +\section*{Data and Code Availability} + +A dataset of the model results is available at \href{zenodoTBA}{doi:zenodoTBA}. +The code to reproduce the experiments is available at \href{https://github.com/repository}{github.com/repository}. + +% tidy with https://flamingtempura.github.io/bibtex-tidy/ +\addcontentsline{toc}{section}{References} +\renewcommand{\ttdefault}{\sfdefault} +\bibliography{references} + +% supplementary information + +\newpage + +\makeatletter +\renewcommand \thesection{S\@arabic\c@section} +\renewcommand\thetable{S\@arabic\c@table} +\renewcommand \thefigure{S\@arabic\c@figure} +\makeatother +\renewcommand{\citenumfont}[1]{S#1} +\setcounter{equation}{0} +\setcounter{figure}{0} +\setcounter{table}{0} +\setcounter{section}{0} + + +\section{Supplementary Information} +\label{sec:si} + +\input{sections/supplementary.tex} + +\addcontentsline{toc}{section}{Supplementary References} +\renewcommand{\ttdefault}{\sfdefault} +\bibliographyS{references} + +\end{document} \ No newline at end of file diff --git a/report/sections/.gitkeep b/report/sections/.gitkeep new file mode 100644 index 000000000..071569975 --- /dev/null +++ b/report/sections/.gitkeep @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Open Energy Transition gGmbH +# +# SPDX-License-Identifier: CC0-1.0 \ No newline at end of file diff --git a/report/sections/abstract.tex b/report/sections/abstract.tex new file mode 100644 index 000000000..bf978e970 --- /dev/null +++ b/report/sections/abstract.tex @@ -0,0 +1,5 @@ +% SPDX-FileCopyrightText: Open Energy Transition gGmbH +% +% SPDX-License-Identifier: MIT + +\lipsum[1] \ No newline at end of file diff --git a/report/sections/conclusion.tex b/report/sections/conclusion.tex new file mode 100644 index 000000000..bf978e970 --- /dev/null +++ b/report/sections/conclusion.tex @@ -0,0 +1,5 @@ +% SPDX-FileCopyrightText: Open Energy Transition gGmbH +% +% SPDX-License-Identifier: MIT + +\lipsum[1] \ No newline at end of file diff --git a/report/sections/discussion.tex b/report/sections/discussion.tex new file mode 100644 index 000000000..bf978e970 --- /dev/null +++ b/report/sections/discussion.tex @@ -0,0 +1,5 @@ +% SPDX-FileCopyrightText: Open Energy Transition gGmbH +% +% SPDX-License-Identifier: MIT + +\lipsum[1] \ No newline at end of file diff --git a/report/sections/introduction.tex b/report/sections/introduction.tex new file mode 100644 index 000000000..bf978e970 --- /dev/null +++ b/report/sections/introduction.tex @@ -0,0 +1,5 @@ +% SPDX-FileCopyrightText: Open Energy Transition gGmbH +% +% SPDX-License-Identifier: MIT + +\lipsum[1] \ No newline at end of file diff --git a/report/sections/methods.tex b/report/sections/methods.tex new file mode 100644 index 000000000..a3c1cd5e9 --- /dev/null +++ b/report/sections/methods.tex @@ -0,0 +1,6 @@ +% SPDX-FileCopyrightText: Open Energy Transition gGmbH +% +% SPDX-License-Identifier: MIT + +\lipsum[1] +Blablabla said Nobody ~\cite{Nobody06}. \ No newline at end of file diff --git a/report/sections/results.tex b/report/sections/results.tex new file mode 100644 index 000000000..bf978e970 --- /dev/null +++ b/report/sections/results.tex @@ -0,0 +1,5 @@ +% SPDX-FileCopyrightText: Open Energy Transition gGmbH +% +% SPDX-License-Identifier: MIT + +\lipsum[1] \ No newline at end of file diff --git a/report/sections/supplementary.tex b/report/sections/supplementary.tex new file mode 100644 index 000000000..bf978e970 --- /dev/null +++ b/report/sections/supplementary.tex @@ -0,0 +1,5 @@ +% SPDX-FileCopyrightText: Open Energy Transition gGmbH +% +% SPDX-License-Identifier: MIT + +\lipsum[1] \ No newline at end of file diff --git a/report/static/.gitkeep b/report/static/.gitkeep new file mode 100644 index 000000000..071569975 --- /dev/null +++ b/report/static/.gitkeep @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Open Energy Transition gGmbH +# +# SPDX-License-Identifier: CC0-1.0 \ No newline at end of file diff --git a/rules/report.smk b/rules/report.smk new file mode 100755 index 000000000..c79d4f0de --- /dev/null +++ b/rules/report.smk @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Open Energy Transition gGmbH +# +# SPDX-License-Identifier: MIT + +rule report: + message: "Compile report." + params: + fn="report" + input: + tex="report/report.tex", + bib="report/references.bib" + output: "report/report.pdf" + shell: + """ + pdflatex -output-directory report {input.tex} + cd report; bibtex {params.fn}; cd .. + pdflatex -output-directory report {input.tex} + pdflatex -output-directory report {input.tex} + """ From 69578e05a69e52239accec1490685ff1e10de305 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 13 Dec 2024 12:24:15 +0100 Subject: [PATCH 7/8] taylor readme to eu-flex purpose --- README.md | 142 +++++++++++++++--------------------------------------- 1 file changed, 38 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index 7ca589ebd..2eb2d11ef 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,39 @@ SPDX-License-Identifier: CC-BY-4.0 [![REUSE status](https://api.reuse.software/badge/github.com/pypsa/pypsa-eur)](https://api.reuse.software/info/github.com/pypsa/pypsa-eur) [![Stack Exchange questions](https://img.shields.io/stackexchange/stackoverflow/t/pypsa)](https://stackoverflow.com/questions/tagged/pypsa) -# {{project_name}} +# EU-Flex Platform Open Energy Transition Logo -This repository is a soft-fork of [PyPSA-Eur](https://github.com/pypsa/pypsa-eur) and contains the entire project `{{project_name}}` supported by [Open Energy Transition (OET)](https://openenergytransition.org/)*, including code and report. The philosophy behind this repository is that no intermediary results are included, but all results are computed from raw data and code. The structure is also inspired by [cookiecutter-project](https://github.com/PyPSA/cookiecutter-project). -This repository is maintained using [OET's soft-fork strategy](https://open-energy-transition.github.io/handbook/docs/Engineering/SoftForkStrategy). OET's primary aim is to contribute as much as possible to the open source (OS) upstream repositories. For long-term changes that cannot be directly merged upstream, the strategy organizes and maintains OET forks, ensuring they remain up-to-date and compatible with upstream, while also supporting future contributions back to the OS repositories. +## Overview +The EU-Flex Platform is an implementation of ACER's EU-wide flexibility assessment platform based on PyPSA-Eur. This project develops a comprehensive modeling tool for assessing power system flexibility needs across the European Union, as mandated by Regulation (EU) 2024/1747. +## Project Scope +This repository covers Phase 1 (Implementation) and Phase 2 (Testing) of the EU-Flex Platform development: -# Repository structure +### Phase 1: Implementation (Dec 2024 - Nov 2025) +- Development of core modeling capabilities using PyPSA +- Implementation of flexibility assessment functionalities +- Integration of various data sources and power system components +- Development of APIs and user interfaces +- Setting up security and operational protocols + +### Phase 2: Testing (Dec 2025 - June 2026) +- Comprehensive testing with complete datasets +- Performance optimization and refinement +- Integration testing with real-world scenarios +- System validation and quality assurance +- Methodology alignment and adjustments + +## Key Features (WIP) +- EU-wide power system modeling +- Flexibility needs assessment across multiple timeframes +- Integration of renewable energy sources +- Analysis of demand-side response and storage options +- Cross-border capacity evaluation +- Stochastic analysis capabilities + +## Repository structure * `benchmarks`: will store `snakemake` benchmarks (does not exist initially) * `config`: configurations used in the study @@ -37,9 +61,14 @@ This repository is maintained using [OET's soft-fork strategy](https://open-ener * `results`: will store the solved PyPSA network data, summary files and plots (does not exist initially) * `scripts`: includes all the Python scripts executed by the `snakemake` rules to build the model -# Installation and Usage +## Technology Stack +- [OET/PyPSA-Eur](https://github.com/open-energy-transition/pypsa-eur) used via the [OET's soft-fork strategy](https://open-energy-transition.github.io/handbook/docs/Engineering/SoftForkStrategy). The model is described in the [documentation](https://pypsa-eur.readthedocs.io) and in the paper +[PyPSA-Eur: An Open Optimisation Model of the European Transmission System](https://arxiv.org/abs/1806.01613), 2018, [arXiv:1806.01613](https://arxiv.org/abs/1806.01613). +- [Snakemake](https://snakemake.readthedocs.io) for workflow management + +## Installation and Usage -## 1. Installation +### 1. Installation Clone the repository: @@ -53,7 +82,7 @@ Activate the newly created `{{project_short_name}}` environment: mamba activate {{project_short_name}} -## 2. Run the analysis +### 2. Run the analysis snakemake -call @@ -65,100 +94,5 @@ To generate a PDF of the dependency graph of all steps `resources/dag.pdf` run: * Open Energy Transition (g)GmbH, Königsallee 52, 95448 Bayreuth, Germany ----- - ----- - -# PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy System - -PyPSA-Eur is an open model dataset of the European energy system at the -transmission network level that covers the full ENTSO-E area. The model is suitable both for operational studies and generation and transmission expansion planning studies. -The continental scope and highly resolved spatial scale enables a proper description of the long-range -smoothing effects for renewable power generation and their varying resource availability. - - - - -The model is described in the [documentation](https://pypsa-eur.readthedocs.io) -and in the paper -[PyPSA-Eur: An Open Optimisation Model of the European Transmission -System](https://arxiv.org/abs/1806.01613), 2018, -[arXiv:1806.01613](https://arxiv.org/abs/1806.01613). -The model building routines are defined through a snakemake workflow. -Please see the [documentation](https://pypsa-eur.readthedocs.io/) -for installation instructions and other useful information about the snakemake workflow. -The model is designed to be imported into the open toolbox -[PyPSA](https://github.com/PyPSA/PyPSA). - -**WARNING**: PyPSA-Eur is under active development and has several -[limitations](https://pypsa-eur.readthedocs.io/en/latest/limitations.html) which -you should understand before using the model. The github repository -[issues](https://github.com/PyPSA/pypsa-eur/issues) collect known topics we are -working on (please feel free to help or make suggestions). The -[documentation](https://pypsa-eur.readthedocs.io/) remains somewhat patchy. You -can find showcases of the model's capabilities in the Joule paper [The potential -role of a hydrogen network in -Europe](https://doi.org/10.1016/j.joule.2023.06.016), another [paper in Joule -with a description of the industry -sector](https://doi.org/10.1016/j.joule.2022.04.016), or in [a 2021 presentation -at EMP-E](https://nworbmot.org/energy/brown-empe.pdf). We do not recommend to -use the full resolution network model for simulations. At high granularity the -assignment of loads and generators to the nearest network node may not be a -correct assumption, depending on the topology of the underlying distribution -grid, and local grid bottlenecks may cause unrealistic load-shedding or -generator curtailment. We recommend to cluster the network to a couple of -hundred nodes to remove these local inconsistencies. See the discussion in -Section 3.4 "Model validation" of the paper. - - -![PyPSA-Eur Grid Model](doc/img/elec.png) - -The dataset consists of: - -- A grid model based on a modified [GridKit](https://github.com/bdw/GridKit) - extraction of the [ENTSO-E Transmission System - Map](https://www.entsoe.eu/data/map/). The grid model contains 7072 lines - (alternating current lines at and above 220kV voltage level and all high - voltage direct current lines) and 3803 substations. -- The open power plant database - [powerplantmatching](https://github.com/PyPSA/powerplantmatching). -- Electrical demand time series from the - [OPSD project](https://open-power-system-data.org/). -- Renewable time series based on ERA5 and SARAH, assembled using the [atlite tool](https://github.com/PyPSA/atlite). -- Geographical potentials for wind and solar generators based on land use (CORINE) and excluding nature reserves (Natura2000) are computed with the [atlite library](https://github.com/PyPSA/atlite). - -A sector-coupled extension adds demand -and supply for the following sectors: transport, space and water -heating, biomass, industry and industrial feedstocks, agriculture, -forestry and fishing. This completes the energy system and includes -all greenhouse gas emitters except waste management and land use. - -This diagram gives an overview of the sectors and the links between -them: - -![sector diagram](doc/img/multisector_figure.png) - -Each of these sectors is built up on the transmission network nodes -from [PyPSA-Eur](https://github.com/PyPSA/pypsa-eur): - -![network diagram](https://github.com/PyPSA/pypsa-eur/blob/master/doc/img/base.png?raw=true) - -For computational reasons the model is usually clustered down -to 50-200 nodes. - -Already-built versions of the model can be found in the accompanying [Zenodo -repository](https://doi.org/10.5281/zenodo.3601881). - -# Contributing and Support -We strongly welcome anyone interested in contributing to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on GitHub. -- In case of code-related **questions**, please post on [stack overflow](https://stackoverflow.com/questions/tagged/pypsa). -- For non-programming related and more general questions please refer to the [mailing list](https://groups.google.com/group/pypsa). -- To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.com/invite/AnuJBk23FU). -- For **bugs and feature requests**, please use the [PyPSA-Eur Github Issues page](https://github.com/PyPSA/pypsa-eur/issues). - -# Licence - -The code in PyPSA-Eur is released as free software under the -[MIT License](https://opensource.org/licenses/MIT), see [`doc/licenses.rst`](doc/licenses.rst). -However, different licenses and terms of use may apply to the various -input data, see [`doc/data_sources.rst`](doc/data_sources.rst). \ No newline at end of file +## Access and Usage +This platform is being developed for ACER (European Union Agency for the Cooperation of Energy Regulators). Access and usage rights are restricted according to ACER's policies and regulations. From 9f1a419a70b9a71a3229971ea550099e4bfd4352 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 19 Dec 2024 09:33:21 +0100 Subject: [PATCH 8/8] remove report.pdf from all --- .gitignore | 9 +++++++++ Snakefile | 1 - 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 43a1d3552..54b36be88 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ gurobi.log /tmp doc/_build + /scripts/old /scripts/create_scenarios.py /config/create_scenarios.py @@ -57,6 +58,14 @@ dask-worker-space/ publications.jrc.ec.europa.eu/ d1gam3xoknrgr2.cloudfront.net/ +#latex files in report/ +report/*.aux +report/*.bbl +report/*.blg +report/*.log +report/*.out +report/*.toc + *.org *.nc diff --git a/Snakefile b/Snakefile index 7be5c20f9..e0db636fd 100644 --- a/Snakefile +++ b/Snakefile @@ -75,7 +75,6 @@ if config["foresight"] == "perfect": rule all: input: expand(RESULTS + "graphs/costs.svg", run=config["run"]["name"]), - "report/report.pdf" default_target: True