Finalize duplication of retrieve_data

Snakefile

@@ -53,12 +53,11 @@ CDIR = RDIR if not run.get("shared_cutouts") else ""
 SECDIR = run["sector_name"] + "/" if run.get("sector_name") else ""
 SDIR = config["summary_dir"].strip("/") + f"/{SECDIR}"
 RESDIR = config["results_dir"].strip("/") + f"/{SECDIR}"
-COSTDIR = config["costs_dir"]
 
 load_data_paths = get_load_paths_gegis("data", config)
 
 if config["enable"].get("retrieve_cost_data", True):
-    COSTS = "resources/" + RDIR + "costs.csv"
+    COSTS = "resources/" + RDIR + f"costs_{config['costs']['year']}.csv"
 else:
     COSTS = "data/costs.csv"
 ATLITE_NPROCESSES = config["atlite"].get("nprocesses", 4)
@@ -392,29 +391,18 @@ if not config["enable"].get("build_natura_raster", False):
 if config["enable"].get("retrieve_cost_data", True):
 
     rule retrieve_cost_data:
+        params:
+            version=config["costs"]["version"],
         input:
             HTTP.remote(
-                f"raw.githubusercontent.com/PyPSA/technology-data/{config['costs']['version']}/outputs/costs_{config['costs']['year']}.csv",
+                f"raw.githubusercontent.com/PyPSA/technology-data/{config['costs']['version']}/outputs/"
+                + "costs_{year}.csv",
                 keep_local=True,
             ),
         output:
-            COSTS,
+            "resources/" + RDIR + "costs_{year}.csv",
         log:
-            "logs/" + RDIR + "retrieve_cost_data.log",
-        resources:
-            mem_mb=5000,
-        run:
-            move(input[0], output[0])
-
-    rule retrieve_cost_data_flexible:
-        input:
-            HTTP.remote(
-                f"raw.githubusercontent.com/PyPSA/technology-data/{config['costs']['version']}/outputs/costs"
-                + "_{planning_horizons}.csv",
-                keep_local=True,
-            ),
-        output:
-            costs=COSTDIR + "costs_{planning_horizons}.csv",
+            "logs/" + RDIR + "retrieve_cost_data_{year}.log",
         resources:
             mem_mb=5000,
         run:
@@ -1071,7 +1059,7 @@ rule prepare_sector_network:
     input:
         network=RESDIR
         + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc",
-        costs=COSTDIR + "costs_{planning_horizons}.csv",
+        costs="resources/" + RDIR + "costs_{planning_horizons}.csv",
         h2_cavern="data/hydrogen_salt_cavern_potentials.csv",
         nodal_energy_totals="resources/"
         + SECDIR
@@ -1163,7 +1151,7 @@ rule add_export:
     input:
         overrides="data/override_component_attrs",
         export_ports="resources/" + SECDIR + "export_ports.csv",
-        costs=COSTDIR + "costs_{planning_horizons}.csv",
+        costs="resources/" + RDIR + "costs_{planning_horizons}.csv",
         ship_profile="resources/" + SECDIR + "ship_profile_{h2export}TWh.csv",
         network=RESDIR
         + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}.nc",
@@ -1621,7 +1609,7 @@ if config["foresight"] == "overnight":
             # + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}.nc",
             network=RESDIR
             + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc",
-            costs=COSTDIR + "costs_{planning_horizons}.csv",
+            costs="resources/" + RDIR + "costs_{planning_horizons}.csv",
             configs=SDIR + "configs/config.yaml",  # included to trigger copy_config rule
         output:
             RESDIR
@@ -1666,7 +1654,7 @@ rule make_sector_summary:
             **config["costs"],
             **config["export"],
         ),
-        costs=COSTDIR + "costs_{planning_horizons}.csv",
+        costs="resources/" + RDIR + "costs_{planning_horizons}.csv",
         plots=expand(
             RESDIR
             + "maps/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}-costs-all_{planning_horizons}_{discountrate}_{demand}_{h2export}export.pdf",
@@ -1903,7 +1891,7 @@ rule build_industry_demand:  #default data
         + SECDIR
         + "demand/base_industry_totals_{planning_horizons}_{demand}.csv",
         industrial_database="data/industrial_database.csv",
-        costs=COSTDIR + "costs_{planning_horizons}.csv",
+        costs="resources/" + RDIR + "costs_{planning_horizons}.csv",
         industry_growth_cagr="data/demand/industry_growth_cagr.csv",
     output:
         industrial_energy_demand_per_node="resources/"

config.default.yaml

@@ -11,7 +11,6 @@ logging:
 
 results_dir: results/
 summary_dir: results/
-costs_dir: data/ # TODO change to the equivalent of technology data
 
 foresight: overnight
 
@@ -357,7 +356,7 @@ renewable:
 # TODO: Needs to be adjusted for Africa.
 costs:
   year: 2030
-  version: v0.6.2
+  version: v0.10.0
   discountrate: [0.071] #, 0.086, 0.111]
   # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html # noqa: E501
   USD2013_to_EUR2013: 0.7532 # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html

doc/release_notes.rst

@@ -15,6 +15,7 @@ This part of documentation collects descriptive release notes to capture the mai
 
 * Include option in the config to allow for custom airport data `PR #1241 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1241>`__
 
+* Drop duplication of retrieve_data and COST_DIR, add params and update technology-data version `PR #1120 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1120>`__
 
 **Minor Changes and bug-fixing**
 

scripts/add_electricity.py

@@ -488,7 +488,10 @@ def attach_hydro(n, costs, ppl):
     ror = ppl.query('technology == "Run-Of-River"')
     phs = ppl.query('technology == "Pumped Storage"')
     hydro = ppl.query('technology == "Reservoir"')
-    bus_id = ppl["bus"]
+    if snakemake.params.alternative_clustering:
+        bus_id = ppl["region_id"]
+    else:
+        bus_id = ppl["bus"]
 
     inflow_idx = ror.index.union(hydro.index)
     if not inflow_idx.empty:

scripts/build_renewable_profiles.py

@@ -356,6 +356,9 @@ def rescale_hydro(plants, runoff, normalize_using_yearly, normalization_year):
         logger.info("No bus has installed hydro plants, ignoring normalization.")
         return runoff
 
+    if snakemake.params.alternative_clustering:
+        plants = plants.set_index("shape_id")
+
     years_statistics = normalize_using_yearly.index
     if isinstance(years_statistics, pd.DatetimeIndex):
         years_statistics = years_statistics.year
@@ -530,6 +533,24 @@ def create_scaling_factor(
         # the region should be restricted for non-hydro technologies, as the hydro potential is calculated across hydrobasins which may span beyond the region of the country
         cutout = filter_cutout_region(cutout, regions)
 
+    if snakemake.params.alternative_clustering:
+        regions = gpd.GeoDataFrame(
+            regions.reset_index()
+            .groupby("shape_id")
+            .agg(
+                {
+                    "x": "mean",
+                    "y": "mean",
+                    "country": "first",
+                    "geometry": "first",
+                    "bus": "first",
+                }
+            )
+            .reset_index()
+            .set_index("bus"),
+            crs=regions.crs,
+        )
+
     buses = regions.index
 
     func = getattr(cutout, resource.pop("method"))
@@ -556,10 +577,17 @@ def create_scaling_factor(
         # select busbar whose location (p) belongs to at least one hydrobasin geometry
         # if extendable option is true, all buses are included
         # otherwise only where hydro powerplants are available are considered
-        filter_bus_to_consider = regions.index.map(
-            lambda bus_id: config.get("extendable", False)
-            | (bus_id in hydro_ppls.bus.values)
-        )
+        if snakemake.params.alternative_clustering:
+            filter_bus_to_consider = regions.index.map(
+                lambda bus_id: config.get("extendable", False)
+                | (bus_id in hydro_ppls.region_id.values)
+            )
+        ### TODO: quickfix. above case and the below case should by unified
+        if snakemake.params.alternative_clustering == False:
+            filter_bus_to_consider = regions.index.map(
+                lambda bus_id: config.get("extendable", False)
+                | (bus_id in hydro_ppls.bus.values)
+            )
         bus_to_consider = regions.index[filter_bus_to_consider]
 
         # identify subset of buses within the hydrobasins
@@ -577,10 +605,17 @@ def create_scaling_factor(
             columns={"x": "lon", "y": "lat", "country": "countries"}
         ).loc[bus_in_hydrobasins, ["lon", "lat", "countries", "shape_id"]]
 
-        resource["plants"]["installed_hydro"] = [
-            True if (bus_id in hydro_ppls.bus.values) else False
-            for bus_id in resource["plants"].index
-        ]
+        # TODO: these cases shall be fixed by restructuring the alternative clustering procedure
+        if snakemake.params.alternative_clustering == False:
+            resource["plants"]["installed_hydro"] = [
+                True if (bus_id in hydro_ppls.bus.values) else False
+                for bus_id in resource["plants"].index
+            ]
+        else:
+            resource["plants"]["installed_hydro"] = [
+                True if (bus_id in hydro_ppls.region_id.values) else False
+                for bus_id in resource["plants"].shape_id.values
+            ]
 
         # get normalization before executing runoff
         normalization = None
@@ -596,6 +631,8 @@ def create_scaling_factor(
         else:
             # otherwise perform the calculations
             inflow = correction_factor * func(capacity_factor=True, **resource)
+            if snakemake.params.alternative_clustering:
+                inflow["plant"] = regions.shape_id.loc[inflow["plant"]].values
 
             if "clip_min_inflow" in config:
                 inflow = inflow.where(inflow >= config["clip_min_inflow"], 0)

test/config.test_myopic.yaml

@@ -8,7 +8,6 @@ tutorial: true
 
 results_dir: results/
 summary_dir: results/
-costs_dir: data/ #TODO change to the equivalent of technology data
 
 run:
   name: "test_myopic" # use this to keep track of runs with different settings
@@ -99,7 +98,7 @@ custom_data:
 
 
 costs: # Costs used in PyPSA-Earth-Sec. Year depends on the wildcard planning_horizon in the scenario section
-  version: v0.6.2
+  version: v0.10.0
   lifetime: 25 #default lifetime
   # From a Lion Hirth paper, also reflects average of Noothout et al 2016
   discountrate: [0.071] #, 0.086, 0.111]