diff --git a/Snakefile b/Snakefile index 5002fe2c5..3b078bd3c 100644 --- a/Snakefile +++ b/Snakefile @@ -53,12 +53,11 @@ CDIR = RDIR if not run.get("shared_cutouts") else "" SECDIR = run["sector_name"] + "/" if run.get("sector_name") else "" SDIR = config["summary_dir"].strip("/") + f"/{SECDIR}" RESDIR = config["results_dir"].strip("/") + f"/{SECDIR}" -COSTDIR = config["costs_dir"] load_data_paths = get_load_paths_gegis("data", config) if config["enable"].get("retrieve_cost_data", True): - COSTS = "resources/" + RDIR + "costs.csv" + COSTS = "resources/" + RDIR + f"costs_{config['costs']['year']}.csv" else: COSTS = "data/costs.csv" ATLITE_NPROCESSES = config["atlite"].get("nprocesses", 4) @@ -392,29 +391,18 @@ if not config["enable"].get("build_natura_raster", False): if config["enable"].get("retrieve_cost_data", True): rule retrieve_cost_data: + params: + version=config["costs"]["version"], input: HTTP.remote( - f"raw.githubusercontent.com/PyPSA/technology-data/{config['costs']['version']}/outputs/costs_{config['costs']['year']}.csv", + f"raw.githubusercontent.com/PyPSA/technology-data/{config['costs']['version']}/outputs/" + + "costs_{year}.csv", keep_local=True, ), output: - COSTS, + "resources/" + RDIR + "costs_{year}.csv", log: - "logs/" + RDIR + "retrieve_cost_data.log", - resources: - mem_mb=5000, - run: - move(input[0], output[0]) - - rule retrieve_cost_data_flexible: - input: - HTTP.remote( - f"raw.githubusercontent.com/PyPSA/technology-data/{config['costs']['version']}/outputs/costs" - + "_{planning_horizons}.csv", - keep_local=True, - ), - output: - costs=COSTDIR + "costs_{planning_horizons}.csv", + "logs/" + RDIR + "retrieve_cost_data_{year}.log", resources: mem_mb=5000, run: @@ -1071,7 +1059,7 @@ rule prepare_sector_network: input: network=RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", - costs=COSTDIR + "costs_{planning_horizons}.csv", + costs="resources/" + RDIR + "costs_{planning_horizons}.csv", h2_cavern="data/hydrogen_salt_cavern_potentials.csv", nodal_energy_totals="resources/" + SECDIR @@ -1165,7 +1153,7 @@ rule add_export: input: overrides="data/override_component_attrs", export_ports="resources/" + SECDIR + "export_ports.csv", - costs=COSTDIR + "costs_{planning_horizons}.csv", + costs="resources/" + RDIR + "costs_{planning_horizons}.csv", ship_profile="resources/" + SECDIR + "ship_profile_{h2export}TWh.csv", network=RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc", @@ -1632,7 +1620,7 @@ if config["foresight"] == "overnight": # + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}.nc", network=RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", - costs=COSTDIR + "costs_{planning_horizons}.csv", + costs="resources/" + RDIR + "costs_{planning_horizons}.csv", configs=SDIR + "configs/config.yaml", # included to trigger copy_config rule output: RESDIR @@ -1677,7 +1665,7 @@ rule make_sector_summary: **config["costs"], **config["export"], ), - costs=COSTDIR + "costs_{planning_horizons}.csv", + costs="resources/" + RDIR + "costs_{planning_horizons}.csv", plots=expand( RESDIR + "maps/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}-costs-all_{planning_horizons}_{discountrate}_{demand}_{h2export}export.pdf", @@ -1915,7 +1903,7 @@ rule build_industry_demand: #default data + SECDIR + "demand/base_industry_totals_{planning_horizons}_{demand}.csv", industrial_database="data/industrial_database.csv", - costs=COSTDIR + "costs_{planning_horizons}.csv", + costs="resources/" + RDIR + "costs_{planning_horizons}.csv", industry_growth_cagr="data/demand/industry_growth_cagr.csv", output: industrial_energy_demand_per_node="resources/" diff --git a/config.default.yaml b/config.default.yaml index ed8554002..5318af083 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -11,7 +11,6 @@ logging: results_dir: results/ summary_dir: results/ -costs_dir: data/ # TODO change to the equivalent of technology data foresight: overnight @@ -358,7 +357,7 @@ renewable: # Costs Configuration costs: year: 2030 - version: v0.6.2 + version: v0.10.0 discountrate: [0.071] #, 0.086, 0.111] # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html # noqa: E501 USD2013_to_EUR2013: 0.7532 # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 80f8f7410..41fbf7819 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -13,6 +13,9 @@ This part of documentation collects descriptive release notes to capture the mai **New Features and Major Changes** +* Drop duplication of retrieve_data and COST_DIR, add params and update technology-data version `PR #1249 `__ + +* In alternative clustering, generate hydro inflows by shape and avoid hydro inflows duplication for plants installed in the same node `PR #1120 ` **Minor Changes and bug-fixing** diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 75ae9ce42..84232f4d8 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -488,7 +488,10 @@ def attach_hydro(n, costs, ppl): ror = ppl.query('technology == "Run-Of-River"') phs = ppl.query('technology == "Pumped Storage"') hydro = ppl.query('technology == "Reservoir"') - bus_id = ppl["bus"] + if snakemake.params.alternative_clustering: + bus_id = ppl["region_id"] + else: + bus_id = ppl["bus"] inflow_idx = ror.index.union(hydro.index) if not inflow_idx.empty: diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 8c16bce09..77427534d 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -356,6 +356,9 @@ def rescale_hydro(plants, runoff, normalize_using_yearly, normalization_year): logger.info("No bus has installed hydro plants, ignoring normalization.") return runoff + if snakemake.params.alternative_clustering: + plants = plants.set_index("shape_id") + years_statistics = normalize_using_yearly.index if isinstance(years_statistics, pd.DatetimeIndex): years_statistics = years_statistics.year @@ -530,6 +533,24 @@ def create_scaling_factor( # the region should be restricted for non-hydro technologies, as the hydro potential is calculated across hydrobasins which may span beyond the region of the country cutout = filter_cutout_region(cutout, regions) + if snakemake.params.alternative_clustering: + regions = gpd.GeoDataFrame( + regions.reset_index() + .groupby("shape_id") + .agg( + { + "x": "mean", + "y": "mean", + "country": "first", + "geometry": "first", + "bus": "first", + } + ) + .reset_index() + .set_index("bus"), + crs=regions.crs, + ) + buses = regions.index func = getattr(cutout, resource.pop("method")) @@ -556,10 +577,17 @@ def create_scaling_factor( # select busbar whose location (p) belongs to at least one hydrobasin geometry # if extendable option is true, all buses are included # otherwise only where hydro powerplants are available are considered - filter_bus_to_consider = regions.index.map( - lambda bus_id: config.get("extendable", False) - | (bus_id in hydro_ppls.bus.values) - ) + if snakemake.params.alternative_clustering: + filter_bus_to_consider = regions.index.map( + lambda bus_id: config.get("extendable", False) + | (bus_id in hydro_ppls.region_id.values) + ) + ### TODO: quickfix. above case and the below case should by unified + if snakemake.params.alternative_clustering == False: + filter_bus_to_consider = regions.index.map( + lambda bus_id: config.get("extendable", False) + | (bus_id in hydro_ppls.bus.values) + ) bus_to_consider = regions.index[filter_bus_to_consider] # identify subset of buses within the hydrobasins @@ -577,10 +605,17 @@ def create_scaling_factor( columns={"x": "lon", "y": "lat", "country": "countries"} ).loc[bus_in_hydrobasins, ["lon", "lat", "countries", "shape_id"]] - resource["plants"]["installed_hydro"] = [ - True if (bus_id in hydro_ppls.bus.values) else False - for bus_id in resource["plants"].index - ] + # TODO: these cases shall be fixed by restructuring the alternative clustering procedure + if snakemake.params.alternative_clustering == False: + resource["plants"]["installed_hydro"] = [ + True if (bus_id in hydro_ppls.bus.values) else False + for bus_id in resource["plants"].index + ] + else: + resource["plants"]["installed_hydro"] = [ + True if (bus_id in hydro_ppls.region_id.values) else False + for bus_id in resource["plants"].shape_id.values + ] # get normalization before executing runoff normalization = None @@ -596,6 +631,8 @@ def create_scaling_factor( else: # otherwise perform the calculations inflow = correction_factor * func(capacity_factor=True, **resource) + if snakemake.params.alternative_clustering: + inflow["plant"] = regions.shape_id.loc[inflow["plant"]].values if "clip_min_inflow" in config: inflow = inflow.where(inflow >= config["clip_min_inflow"], 0) diff --git a/test/config.test_myopic.yaml b/test/config.test_myopic.yaml index 382def55f..ab306a63a 100644 --- a/test/config.test_myopic.yaml +++ b/test/config.test_myopic.yaml @@ -8,7 +8,6 @@ tutorial: true results_dir: results/ summary_dir: results/ -costs_dir: data/ #TODO change to the equivalent of technology data run: name: "test_myopic" # use this to keep track of runs with different settings @@ -99,7 +98,7 @@ custom_data: costs: # Costs used in PyPSA-Earth-Sec. Year depends on the wildcard planning_horizon in the scenario section - version: v0.6.2 + version: v0.10.0 lifetime: 25 #default lifetime # From a Lion Hirth paper, also reflects average of Noothout et al 2016 discountrate: [0.071] #, 0.086, 0.111]