From 057efd1d7860fa0b310f99a692858e5ad07b3270 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Fri, 25 Oct 2024 15:24:08 -0700 Subject: [PATCH 01/27] add generation shares --- docs/api/osemosys_global.demand.rst | 85 +++++++++++++++++++ .../{ => summary}/carbon_intensity.py | 0 .../osemosys_global/summary/constants.py | 5 ++ .../osemosys_global/summary/gen_shares.py | 77 +++++++++++++++++ .../{ => summary}/summarise_results.py | 0 5 files changed, 167 insertions(+) create mode 100644 docs/api/osemosys_global.demand.rst rename workflow/scripts/osemosys_global/{ => summary}/carbon_intensity.py (100%) create mode 100644 workflow/scripts/osemosys_global/summary/constants.py create mode 100644 workflow/scripts/osemosys_global/summary/gen_shares.py rename workflow/scripts/osemosys_global/{ => summary}/summarise_results.py (100%) diff --git a/docs/api/osemosys_global.demand.rst b/docs/api/osemosys_global.demand.rst new file mode 100644 index 00000000..8eb9e635 --- /dev/null +++ b/docs/api/osemosys_global.demand.rst @@ -0,0 +1,85 @@ +osemosys\_global.demand package +=============================== + +Submodules +---------- + +osemosys\_global.demand.constants module +---------------------------------------- + +.. automodule:: osemosys_global.demand.constants + :members: + :undoc-members: + :show-inheritance: + +osemosys\_global.demand.custom module +------------------------------------- + +.. automodule:: osemosys_global.demand.custom + :members: + :undoc-members: + :show-inheritance: + +osemosys\_global.demand.data module +----------------------------------- + +.. automodule:: osemosys_global.demand.data + :members: + :undoc-members: + :show-inheritance: + +osemosys\_global.demand.figures module +-------------------------------------- + +.. automodule:: osemosys_global.demand.figures + :members: + :undoc-members: + :show-inheritance: + +osemosys\_global.demand.main module +----------------------------------- + +.. automodule:: osemosys_global.demand.main + :members: + :undoc-members: + :show-inheritance: + +osemosys\_global.demand.projection module +----------------------------------------- + +.. automodule:: osemosys_global.demand.projection + :members: + :undoc-members: + :show-inheritance: + +osemosys\_global.demand.read module +----------------------------------- + +.. automodule:: osemosys_global.demand.read + :members: + :undoc-members: + :show-inheritance: + +osemosys\_global.demand.regression module +----------------------------------------- + +.. automodule:: osemosys_global.demand.regression + :members: + :undoc-members: + :show-inheritance: + +osemosys\_global.demand.spatial module +-------------------------------------- + +.. automodule:: osemosys_global.demand.spatial + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: osemosys_global.demand + :members: + :undoc-members: + :show-inheritance: diff --git a/workflow/scripts/osemosys_global/carbon_intensity.py b/workflow/scripts/osemosys_global/summary/carbon_intensity.py similarity index 100% rename from workflow/scripts/osemosys_global/carbon_intensity.py rename to workflow/scripts/osemosys_global/summary/carbon_intensity.py diff --git a/workflow/scripts/osemosys_global/summary/constants.py b/workflow/scripts/osemosys_global/summary/constants.py new file mode 100644 index 00000000..475c5fb2 --- /dev/null +++ b/workflow/scripts/osemosys_global/summary/constants.py @@ -0,0 +1,5 @@ +"""Constants for summary statistics""" + +RENEWABLES = ["GEO", "HYD", "SPV", "CSP", "WAS", "WAV", "WON", "WOF"] +CLEAN = ["BIO", "GEO", "HYD", "SPV", "CSP", "WAS", "WAV", "WON", "WOF", "URN"] +FOSSIL = ["CCG", "COA", "COG", "OCG", "OIL", "OTH", "PET"] diff --git a/workflow/scripts/osemosys_global/summary/gen_shares.py b/workflow/scripts/osemosys_global/summary/gen_shares.py new file mode 100644 index 00000000..6f1b38ea --- /dev/null +++ b/workflow/scripts/osemosys_global/summary/gen_shares.py @@ -0,0 +1,77 @@ +"""Calcualtes Renewable Generation + +Since emissions are reported at country levels, carbon intensities are also +reported at country levels. +""" + +import pandas as pd +from typing import Optional +from constants import CLEAN, RENEWABLES, FOSSIL + + +def _get_gen_by_node( + production: pd.DataFrame, carriers: Optional[list[str]] = None +) -> pd.DataFrame: + + df = production.copy() + + assert "TECHNOLOGY" in df.index.names + + df["TECH"] = df.index.get_level_values("TECHNOLOGY").str[3:6] + df["NODE"] = df.index.get_level_values("TECHNOLOGY").str[6:11] + + if carriers: + df = df[df.TECH.isin(carriers)] + + return ( + df.reset_index()[["REGION", "NODE", "YEAR", "VALUE"]] + .groupby(["REGION", "NODE", "YEAR"]) + .sum() + ) + + +def calc_generation_shares(production_by_technology: pd.DataFrame) -> pd.DataFrame: + + df = production_by_technology.copy() + + df = df[ + (df.index.get_level_values("TECHNOLOGY").str.startswith("PWR")) + & ~(df.index.get_level_values("TECHNOLOGY").str.contains("TRN")) + ] + + total = _get_gen_by_node(df).rename(columns={"VALUE": "TOTAL"}) + clean = _get_gen_by_node(df, CLEAN).rename(columns={"VALUE": "CLEAN"}) + renewable = _get_gen_by_node(df, RENEWABLES).rename(columns={"VALUE": "RENEWABLE"}) + fossil = _get_gen_by_node(df, FOSSIL).rename(columns={"VALUE": "FOSSIL"}) + + shares = ( + total.join(clean, how="outer") + .join(renewable, how="outer") + .join(fossil, how="outer") + .fillna(0) + ) + + shares["CLEAN"] = shares.CLEAN.div(shares.TOTAL).mul(100) + shares["RENEWABLE"] = shares.RENEWABLE.div(shares.TOTAL).mul(100) + shares["FOSSIL"] = shares.FOSSIL.div(shares.TOTAL).mul(100) + + return shares[["CLEAN", "RENEWABLE", "FOSSIL"]].round(1) + + +if __name__ == "__main__": + if "snakemake" in globals(): + production_by_technology_annual_csv = snakemake.input.production_by_technology + save = snakemake.output.generation_shares + else: + production_by_technology_annual_csv = ( + "results/India/results/ProductionByTechnologyAnnual.csv" + ) + save = "results/India/results/GenerationShares.csv" + + production_by_technology_annual = pd.read_csv( + production_by_technology_annual_csv, index_col=[0, 1, 2, 3] + ) + + df = calc_generation_shares(production_by_technology_annual) + + df.to_csv(save, index=True) diff --git a/workflow/scripts/osemosys_global/summarise_results.py b/workflow/scripts/osemosys_global/summary/summarise_results.py similarity index 100% rename from workflow/scripts/osemosys_global/summarise_results.py rename to workflow/scripts/osemosys_global/summary/summarise_results.py From 704453476ce90ea291af2dafa22a8b23abba4322 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sat, 26 Oct 2024 18:03:55 -0700 Subject: [PATCH 02/27] node cost --- .../osemosys_global/summary/node_cost.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 workflow/scripts/osemosys_global/summary/node_cost.py diff --git a/workflow/scripts/osemosys_global/summary/node_cost.py b/workflow/scripts/osemosys_global/summary/node_cost.py new file mode 100644 index 00000000..7244a759 --- /dev/null +++ b/workflow/scripts/osemosys_global/summary/node_cost.py @@ -0,0 +1,61 @@ +"""Gets power generation cost per node""" + +import pandas as pd + +def get_cost_per_node(discounted_cost_tech: pd.DataFrame) -> pd.DataFrame: + """Only of power generation technologies""" + + df = discounted_cost_tech.copy() + + df = df[ + (df.index.get_level_values("TECHNOLOGY").str.startswith("PWR")) + & ~(df.index.get_level_values("TECHNOLOGY").str.contains("TRN")) + ] + + df["NODE"] = df.index.get_level_values("TECHNOLOGY").str[6:11] + + return df.reset_index()[["REGION", "NODE", "YEAR", "VALUE"]].groupby(["REGION", "NODE", "YEAR"]).sum() + +def get_demand_per_node(demand: pd.DataFrame) -> pd.DataFrame: + + df = demand.copy() + + df = df[df.index.get_level_values("FUEL").str.startswith("ELC")] + df["NODE"] = df.index.get_level_values("FUEL").str[3:8] + + return df.reset_index()[["REGION", "NODE", "YEAR", "VALUE"]].groupby(["REGION", "NODE", "YEAR"]).sum() + + +def get_pwr_cost_per_node(demand: pd.DataFrame, cost: pd.DataFrame) -> pd.DataFrame: + """Gets power generation cost per node in $/MWh""" + + # ($M / PJ) (1PJ / 1000 TJ) (1TJ / 1000 GJ) (1GJ / 1000 MJ) ($1000000 / $M) (3600sec / hr) + df = cost.div(demand) + return df.mul(3.6) + + +if __name__ == "__main__": + if "snakemake" in globals(): + discounted_cost_by_technology_csv = snakemake.input.discounted_cost_by_technology + demand_csv = snakemake.input.demand + save = snakemake.output.node_cost + else: + discounted_cost_by_technology_csv = ( + "results/India/results/DiscountedCostByTechnology.csv" + ) + demand_csv = "results/India/results/Demand.csv" + save = "results/India/results/NodeCost.csv" + + discounted_cost_by_technology_raw = pd.read_csv( + discounted_cost_by_technology_csv, index_col=[0, 1, 2] + ) + demand_raw = pd.read_csv( + demand_csv, index_col=[0, 1, 2, 3] + ) + + cost = get_cost_per_node(discounted_cost_by_technology_raw) + demand = get_demand_per_node(demand_raw) + + df = get_pwr_cost_per_node(demand, cost) + + df.to_csv(save, index=True) From 84f297db83f7f1b4fea054ba8939acbdac21ba1f Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 27 Oct 2024 20:22:57 -0700 Subject: [PATCH 03/27] summaries added --- resources/otoole.yaml | 111 +--------- workflow/rules/postprocess.smk | 112 ++++++++--- workflow/rules/validate.smk | 4 +- .../osemosys_global/summary/capacity.py | 49 +++++ .../osemosys_global/summary/constants.py | 30 +++ .../osemosys_global/summary/gen_shares.py | 6 +- .../osemosys_global/summary/trade_flows.py | 190 ++++++++++++++++++ workflow/snakefile | 5 +- 8 files changed, 363 insertions(+), 144 deletions(-) create mode 100644 workflow/scripts/osemosys_global/summary/capacity.py create mode 100644 workflow/scripts/osemosys_global/summary/trade_flows.py diff --git a/resources/otoole.yaml b/resources/otoole.yaml index 7219f635..295483cb 100644 --- a/resources/otoole.yaml +++ b/resources/otoole.yaml @@ -307,286 +307,187 @@ AnnualEmissions: type: result dtype: float default: 0 - calculated: True AccumulatedNewCapacity: indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 - calculated: True -AccumulatedNewStorageCapacity: - indices: [REGION, STORAGE, YEAR] - type: result - dtype: float - default: 0 - calculated: True AnnualFixedOperatingCost: indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 - calculated: True AnnualTechnologyEmission: indices: [REGION, TECHNOLOGY, EMISSION, YEAR] type: result dtype: float default: 0 - calculated: True -AnnualTechnologyEmissionsPenalty: - indices: [REGION, TECHNOLOGY, YEAR] - type: result - dtype: float - default: 0 - short_name: AnnualTechEmissionsPenalty - calculated: True AnnualTechnologyEmissionByMode: indices: [REGION, TECHNOLOGY, EMISSION, MODE_OF_OPERATION, YEAR] type: result dtype: float default: 0 - calculated: True AnnualVariableOperatingCost: indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 - calculated: True CapitalInvestment: indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 - calculated: True -CapitalInvestmentStorage: - indices: [REGION, STORAGE, YEAR] - type: result - dtype: float - default: 0 - calculated: True Demand: indices: [REGION, TIMESLICE, FUEL, YEAR] type: result dtype: float default: 0 - calculated: True -DiscountedCapitalInvestmentStorage: - indices: [REGION, STORAGE, YEAR] - type: result - dtype: float - default: 0 - short_name: DiscountedCapexStorage - calculated: False -DiscountedOperatingCost: +DiscountedCapitalInvestment: indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 - calculated: False -DiscountedSalvageValue: +DiscountedCostByTechnology: indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 - calculated: False -DiscountedSalvageValueStorage: - indices: [REGION, STORAGE, YEAR] +DiscountedOperationalCost: + indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 - calculated: False -TotalDiscountedStorageCost: - indices: [REGION, STORAGE, YEAR] +DiscountedSalvageValue: + indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 - calculated: False DiscountedTechnologyEmissionsPenalty: short_name: DiscountedTechEmissionsPenalty indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 - calculated: False -ModelPeriodEmissions: - indices: [REGION, EMISSION] - type: result - dtype: float - default: 0 - calculated: False -NetChargeWithinDay: - indices: [REGION, STORAGE, SEASON, DAYTYPE, DAILYTIMEBRACKET, YEAR] - type: result - dtype: float - default: 0 - calculated: False -NetChargeWithinYear: - indices: [REGION, STORAGE, SEASON, DAYTYPE, DAILYTIMEBRACKET, YEAR] - type: result - dtype: float - default: 0 - calculated: False NewCapacity: indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 - calculated: False NewStorageCapacity: indices: [REGION, STORAGE, YEAR] type: result dtype: float default: 0 - calculated: False NumberOfNewTechnologyUnits: indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 - calculated: False -OperatingCost: - indices: [REGION, TECHNOLOGY, YEAR] - type: result - dtype: float - default: 0 - calculated: False ProductionByTechnology: indices: [REGION, TIMESLICE, TECHNOLOGY, FUEL, YEAR] type: result dtype: float default: 0 - calculated: True ProductionByTechnologyAnnual: indices: [REGION, TECHNOLOGY, FUEL, YEAR] type: result dtype: float default: 0 - calculated: True RateOfActivity: indices: [REGION, TIMESLICE, TECHNOLOGY, MODE_OF_OPERATION, YEAR] type: result dtype: float default: 0 - calculated: False RateOfProductionByTechnology: indices: [REGION, TIMESLICE, TECHNOLOGY, FUEL, YEAR] type: result dtype: float default: 0 - calculated: True RateOfProductionByTechnologyByMode: short_name: RateOfProductionByTechByMode indices: [REGION, TIMESLICE, TECHNOLOGY, MODE_OF_OPERATION, FUEL, YEAR] type: result dtype: float default: 0 - calculated: True RateOfUseByTechnology: indices: [REGION, TIMESLICE, TECHNOLOGY, FUEL, YEAR] type: result dtype: float default: 0 - calculated: True RateOfUseByTechnologyByMode: indices: [REGION, TIMESLICE, TECHNOLOGY, MODE_OF_OPERATION, FUEL, YEAR] type: result dtype: float default: 0 - calculated: True SalvageValue: indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 - calculated: False SalvageValueStorage: indices: [REGION, STORAGE, YEAR] type: result dtype: float default: 0 - calculated: False StorageLevelDayTypeFinish: indices: [REGION, STORAGE, SEASON, DAYTYPE, YEAR] type: result dtype: float default: 0 - calculated: False StorageLevelDayTypeStart: indices: [REGION, STORAGE, SEASON, DAYTYPE, YEAR] type: result dtype: float default: 0 - calculated: False StorageLevelSeasonStart: indices: [REGION, STORAGE, SEASON, YEAR] type: result dtype: float default: 0 - calculated: False StorageLevelYearStart: indices: [REGION, STORAGE, YEAR] type: result dtype: float default: 0 - calculated: False StorageLevelYearFinish: indices: [REGION, STORAGE, YEAR] type: result dtype: float default: 0 - calculated: False -StorageLowerLimit: - indices: [REGION, STORAGE, YEAR] - type: result - dtype: float - default: 0 - calculated: False -StorageUpperLimit: - indices: [REGION, STORAGE, YEAR] - type: result - dtype: float - default: 0 - calculated: False TotalAnnualTechnologyActivityByMode: short_name: TotalAnnualTechActivityByMode indices: [REGION, TECHNOLOGY, MODE_OF_OPERATION, YEAR] type: result dtype: float default: 0 - calculated: True TotalCapacityAnnual: indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 - calculated: True TotalDiscountedCost: indices: [REGION,YEAR] type: result dtype: float default: 0 - calculated: True TotalTechnologyAnnualActivity: indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 - calculated: True TotalTechnologyModelPeriodActivity: short_name: TotalTechModelPeriodActivity indices: [REGION, TECHNOLOGY] type: result dtype: float default: 0 - calculated: True Trade: indices: [REGION, REGION, TIMESLICE, FUEL, YEAR] type: result dtype: float default: 0 - calculated: False UseByTechnology: indices: [REGION, TIMESLICE, TECHNOLOGY, FUEL, YEAR] type: result dtype: float default: 0 - calculated: False diff --git a/workflow/rules/postprocess.smk b/workflow/rules/postprocess.smk index 1265f865..13d6af26 100644 --- a/workflow/rules/postprocess.smk +++ b/workflow/rules/postprocess.smk @@ -2,37 +2,37 @@ import os # OUTPUT FILES -result_figures = [ +RESULT_FIGURES = [ 'TotalCapacityAnnual', 'GenerationAnnual', ] -result_summaries = [ - 'Metrics' +RESULT_SUMMARIES = [ + "TradeFlows", + "AnnualEmissionIntensity", + "TransmissionCapacity", + "PowerCapacity", + "GenerationShares", + "NodeCost" ] -# rules - rule otoole_results: message: 'Generating result csv files...' input: solution_file = "results/{scenario}/{scenario}.sol", - datafile = 'results/{scenario}/{scenario}.txt', otoole_config = 'results/{scenario}/otoole.yaml', + params: + csv_dir = 'results/{scenario}/data', output: expand('results/{{scenario}}/results/{result_file}.csv', result_file = OTOOLE_RESULTS), - # conda: - # '../envs/otoole.yaml' log: log = 'results/{scenario}/logs/otoole_results.log' shell: """ otoole results {config[solver]} csv \ {input.solution_file} results/{wildcards.scenario}/results \ - datafile {input.datafile} \ - {input.otoole_config} - 2> {log} + csv {params.csv_dir} {input.otoole_config} 2> {log} """ rule visualisation: @@ -49,29 +49,45 @@ rule visualisation: results_by_country = config['results_by_country'], years = [config['endYear']], output: - expand('results/{{scenario}}/figures/{result_figure}.html', result_figure = result_figures) + expand('results/{{scenario}}/figures/{result_figure}.html', result_figure = RESULT_FIGURES) log: log = 'results/{scenario}/logs/visualisation.log' shell: 'python workflow/scripts/osemosys_global/visualise.py {params.input_data} {params.result_data} {params.scenario_figs_dir} {params.cost_line_expansion_xlsx} {params.countries} {params.results_by_country} {params.years} 2> {log}' -rule summarise_results: - message: - 'Generating summary of results...' - input: - csv_files = expand('results/{{scenario}}/results/{result_file}.csv', result_file = OTOOLE_RESULTS), +# rule summarise_results: +# message: +# 'Generating summary of results...' +# input: +# csv_files = expand('results/{{scenario}}/results/{result_file}.csv', result_file = OTOOLE_RESULTS), +# params: +# start_year = config['startYear'], +# end_year = config['endYear'], +# dayparts = config['dayparts'], +# seasons = config['seasons'], +# output: +# expand('results/{{scenario}}/result_summaries/{result_summary}.csv', +# result_summary = result_summaries), +# log: +# log = 'results/{scenario}/logs/summarise_results.log' +# shell: +# 'python workflow/scripts/osemosys_global/summarise_results.py 2> {log}' + +rule calculate_trade_flows: + message: + "Calculating Hourly Trade Flows" params: - start_year = config['startYear'], - end_year = config['endYear'], - dayparts = config['dayparts'], - seasons = config['seasons'], + seasons = config["seasons"], + dayparts = config["dayparts"], + timeshift = config["timeshift"], + input: + activity_by_mode = "results/{scenario}/results/TotalAnnualTechnologyActivityByMode.csv", output: - expand('results/{{scenario}}/result_summaries/{result_summary}.csv', - result_summary = result_summaries), + trade_flows = "results/{scenario}/result_summaries/TradeFlows.csv", log: - log = 'results/{scenario}/logs/summarise_results.log' - shell: - 'python workflow/scripts/osemosys_global/summarise_results.py 2> {log}' + log = "results/{scenario}/logs/trade_flows.log" + script: + "../scripts/osemosys_global/summary/trade_flows.py" rule calculate_carbon_intensity: message: @@ -80,8 +96,46 @@ rule calculate_carbon_intensity: production_by_technology = "results/{scenario}/results/ProductionByTechnologyAnnual.csv", annual_emissions = "results/{scenario}/results/AnnualEmissions.csv", output: - emission_intensity = "results/{scenario}/results/AnnualEmissionIntensity.csv", + emission_intensity = "results/{scenario}/result_summaries/AnnualEmissionIntensity.csv", + log: + log = 'results/{scenario}/logs/carbon_intensity.log' + script: + "../scripts/osemosys_global/summary/carbon_intensity.py" + +rule calculate_cost_by_node: + message: + "Calculating Cost by Node..." + input: + discounted_cost_by_technology = "results/{scenario}/results/DiscountedCostByTechnology.csv", + demand = "results/{scenario}/results/Demand.csv", + output: + node_cost = "results/{scenario}/result_summaries/NodeCost.csv", + log: + log = 'results/{scenario}/logs/node_cost.log' + script: + "../scripts/osemosys_global/summary/node_cost.py" + +rule calculate_generation_shares: + message: + "Calculating Generaion Fuel Shares..." + input: + production_by_technology = "results/{scenario}/results/ProductionByTechnology.csv", + output: + generation_shares = "results/{scenario}/result_summaries/GenerationShares.csv", + log: + log = 'results/{scenario}/logs/generation_shares.log' + script: + "../scripts/osemosys_global/summary/gen_shares.py" + +rule calculate_capacity_by_node: + message: + "Calculating Capacity by Node..." + input: + total_capacity = "results/{scenario}/results/TotalCapacityAnnual.csv", + output: + power_capacity = "results/{scenario}/result_summaries/PowerCapacity.csv", + transmission_capacity = "results/{scenario}/result_summaries/TransmissionCapacity.csv", log: - log = 'results/{scenario}/logs/calcualte_carbon_intensity.log' + log = 'results/{scenario}/logs/generation_shares.log' script: - "../scripts/osemosys_global/carbon_intensity.py" \ No newline at end of file + "../scripts/osemosys_global/summary/capacity.py" \ No newline at end of file diff --git a/workflow/rules/validate.smk b/workflow/rules/validate.smk index 11dcfdd8..301289b7 100644 --- a/workflow/rules/validate.smk +++ b/workflow/rules/validate.smk @@ -109,9 +109,9 @@ rule validate_emission_intensity: message: "Validating emission intensity against {wildcards.datasource}" input: validation_data = emission_intensity_validation_data, - og_result = "results/{scenario}/results/AnnualEmissionIntensity.csv" + og_result = "results/{scenario}/result_summaries/AnnualEmissionIntensity.csv" params: - result_dir="results/{scenario}/results", + result_dir="results/{scenario}/result_summaries", variable="emission_intensity" output: expand("results/{{scenario}}/validation/{country}/emission_intensity/{{datasource}}.png", country=COUNTRIES) diff --git a/workflow/scripts/osemosys_global/summary/capacity.py b/workflow/scripts/osemosys_global/summary/capacity.py new file mode 100644 index 00000000..a73849cd --- /dev/null +++ b/workflow/scripts/osemosys_global/summary/capacity.py @@ -0,0 +1,49 @@ +"""Summarizes tranmsission and power capacity""" + +import pandas as pd + +def calc_trn_capacity(total_capacity_annual: pd.DataFrame) -> pd.DataFrame: + + df = total_capacity_annual.copy() + + df = df[df.index.get_level_values("TECHNOLOGY").str.startswith("TRN")] + df["FROM"] = df.index.get_level_values("TECHNOLOGY").str[3:8] + df["TO"] = df.index.get_level_values("TECHNOLOGY").str[8:13] + + return df.reset_index()[["FROM", "TO", "YEAR", "VALUE"]].groupby(["FROM", "TO", "YEAR"]).sum() + +def calc_pwr_capacity(total_capacity_annual: pd.DataFrame) -> pd.DataFrame: + + df = total_capacity_annual.copy() + + df = df[ + (df.index.get_level_values("TECHNOLOGY").str.startswith("PWR")) + & ~(df.index.get_level_values("TECHNOLOGY").str.contains("TRN")) + ] + + df["TECH"] = df.index.get_level_values("TECHNOLOGY").str[3:6] + df["NODE"] = df.index.get_level_values("TECHNOLOGY").str[6:11] + + return df.reset_index()[["TECH", "NODE", "YEAR", "VALUE"]].groupby(["TECH", "NODE", "YEAR"]).sum().rename(columns={"TECH": "TECHNOLOGY"}) + +if __name__ == "__main__": + if "snakemake" in globals(): + total_capacity_csv = snakemake.input.total_capacity + pwr_save = snakemake.output.power_capacity + trn_save = snakemake.output.transmission_capacity + else: + total_capacity_csv = ( + "results/India/results/TotalCapacityAnnual.csv" + ) + pwr_save = "" + trn_save = "" + + total_capacity_annual = pd.read_csv( + total_capacity_csv, index_col=[0, 1, 2] + ) + + pwr_capacity = calc_pwr_capacity(total_capacity_annual) + trn_capacity = calc_trn_capacity(total_capacity_annual) + + pwr_capacity.to_csv(pwr_save, index=True) + trn_capacity.to_csv(trn_save, index=True) diff --git a/workflow/scripts/osemosys_global/summary/constants.py b/workflow/scripts/osemosys_global/summary/constants.py index 475c5fb2..5b5bc37b 100644 --- a/workflow/scripts/osemosys_global/summary/constants.py +++ b/workflow/scripts/osemosys_global/summary/constants.py @@ -3,3 +3,33 @@ RENEWABLES = ["GEO", "HYD", "SPV", "CSP", "WAS", "WAV", "WON", "WOF"] CLEAN = ["BIO", "GEO", "HYD", "SPV", "CSP", "WAS", "WAV", "WON", "WOF", "URN"] FOSSIL = ["CCG", "COA", "COG", "OCG", "OIL", "OTH", "PET"] + +MONTH_NAMES = { + 1: 'Jan', + 2: 'Feb', + 3: 'Mar', + 4: 'Apr', + 5: 'May', + 6: 'Jun', + 7: 'Jul', + 8: 'Aug', + 9: 'Sep', + 10: 'Oct', + 11: 'Nov', + 12: 'Dec', +} + +DAYS_PER_MONTH = { + 'Jan': 31, + 'Feb': 28, + 'Mar': 31, + 'Apr': 30, + 'May': 31, + 'Jun': 30, + 'Jul': 31, + 'Aug': 31, + 'Sep': 30, + 'Oct': 31, + 'Nov': 30, + 'Dec': 31, +} \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/summary/gen_shares.py b/workflow/scripts/osemosys_global/summary/gen_shares.py index 6f1b38ea..1750330e 100644 --- a/workflow/scripts/osemosys_global/summary/gen_shares.py +++ b/workflow/scripts/osemosys_global/summary/gen_shares.py @@ -1,8 +1,4 @@ -"""Calcualtes Renewable Generation - -Since emissions are reported at country levels, carbon intensities are also -reported at country levels. -""" +"""Calcualtes Renewable Generation""" import pandas as pd from typing import Optional diff --git a/workflow/scripts/osemosys_global/summary/trade_flows.py b/workflow/scripts/osemosys_global/summary/trade_flows.py new file mode 100644 index 00000000..5fe3116d --- /dev/null +++ b/workflow/scripts/osemosys_global/summary/trade_flows.py @@ -0,0 +1,190 @@ +"""Calcualtes Transmission Capacity per node""" + +import pandas as pd +import itertools +from constants import MONTH_NAMES, DAYS_PER_MONTH + + +def apply_timeshift(x: int, timeshift: int) -> int: + """Applies timeshift to organize dayparts. + + Arguments: + x = Value between 0-24 + timeshift = value offset from UTC (-11 -> +12) + """ + + x += timeshift + if x > 23: + return x - 24 + elif x < 0: + return x + 24 + else: + return x + + +def get_trade_flows( + activity_by_mode: pd.DataFrame, + seasons_raw: dict[str, list[int]], + dayparts_raw: dict[str, list[int]], + timeshift: int, +) -> pd.DataFrame: + + abm = activity_by_mode.copy() + + interconnections = ( + abm[abm.index.get_level_values("TECHNOLOGY").str.startswith("TRN")] + .index.get_level_values("TECHNOLOGY") + .unique() + .tolist() + ) + + years = abm.index.get_level_values("YEAR").unique().tolist() + + if len(interconnections) > 0: + + seasonsData = [] + + for s, months in seasons_raw.items(): + for month in months: + seasonsData.append([month, s]) + seasons_df = pd.DataFrame(seasonsData, columns=["month", "season"]) + seasons_df = seasons_df.sort_values(by=["month"]).reset_index(drop=True) + + daypartData = [] + for dp, hr in dayparts_raw.items(): + daypartData.append([dp, hr[0], hr[1]]) + dayparts_df = pd.DataFrame( + daypartData, columns=["daypart", "start_hour", "end_hour"] + ) + + dayparts_df["start_hour"] = dayparts_df["start_hour"].map( + lambda x: apply_timeshift(x, timeshift) + ) + dayparts_df["end_hour"] = dayparts_df["end_hour"].map( + lambda x: apply_timeshift(x, timeshift) + ) + + month_names = MONTH_NAMES + days_per_month = DAYS_PER_MONTH + + seasons_df["month_name"] = seasons_df["month"].map(month_names) + seasons_df["days"] = seasons_df["month_name"].map(days_per_month) + seasons_df_grouped = seasons_df.groupby(["season"], as_index=False)[ + "days" + ].sum() + days_dict = dict( + zip(list(seasons_df_grouped["season"]), list(seasons_df_grouped["days"])) + ) + seasons_df["days"] = seasons_df["season"].map(days_dict) + + seasons_dict = dict(zip(list(seasons_df["month"]), list(seasons_df["season"]))) + + dayparts_dict = { + i: [j, k] + for i, j, k in zip( + list(dayparts_df["daypart"]), + list(dayparts_df["start_hour"]), + list(dayparts_df["end_hour"]), + ) + } + + hours_dict = { + i: abs(k - j) + for i, j, k in zip( + list(dayparts_df["daypart"]), + list(dayparts_df["start_hour"]), + list(dayparts_df["end_hour"]), + ) + } + + months = list(seasons_dict) + hours = list(range(1, 25)) + + # APPLY TRANSFORMATION + + df_ts_template = pd.DataFrame( + list(itertools.product(interconnections, months, hours, years)), + columns=["TECHNOLOGY", "MONTH", "HOUR", "YEAR"], + ) + + df_ts_template = df_ts_template.sort_values(by=["TECHNOLOGY", "YEAR"]) + df_ts_template["SEASON"] = df_ts_template["MONTH"].map(seasons_dict) + df_ts_template["DAYS"] = df_ts_template["SEASON"].map(days_dict) + df_ts_template["YEAR"] = df_ts_template["YEAR"].astype(int) + + for daypart in dayparts_dict: + if ( + dayparts_dict[daypart][0] > dayparts_dict[daypart][1] + ): # loops over 24hrs + df_ts_template.loc[ + (df_ts_template["HOUR"] >= dayparts_dict[daypart][0]) + | (df_ts_template["HOUR"] < dayparts_dict[daypart][1]), + "DAYPART", + ] = daypart + else: + df_ts_template.loc[ + (df_ts_template["HOUR"] >= dayparts_dict[daypart][0]) + & (df_ts_template["HOUR"] < dayparts_dict[daypart][1]), + "DAYPART", + ] = daypart + + df_ts_template = df_ts_template.drop_duplicates() + + # Trade flows + df = abm.copy().reset_index() + + df["SEASON"] = df["TIMESLICE"].str[0:2] + df["DAYPART"] = df["TIMESLICE"].str[2:] + df["YEAR"] = df["YEAR"].astype(int) + df.drop(["REGION", "TIMESLICE"], axis=1, inplace=True) + + df = pd.merge( + df, + df_ts_template, + how="left", + on=["TECHNOLOGY", "SEASON", "DAYPART", "YEAR"], + ).dropna() + + df["HOUR_COUNT"] = df["DAYPART"].map(hours_dict) + df["VALUE"] = (df["VALUE"].mul(1e6)) / (df["DAYS"] * df["HOUR_COUNT"].mul(3600)) + + df = df[["YEAR", "MONTH", "HOUR", "TECHNOLOGY", "MODE_OF_OPERATION", "VALUE"]] + df["MODE_OF_OPERATION"] = df["MODE_OF_OPERATION"].astype(int) + df.loc[df["MODE_OF_OPERATION"] == 2, "VALUE"] *= -1 + + df["NODE_1"] = df.TECHNOLOGY.str[3:8] + df["NODE_2"] = df.TECHNOLOGY.str[8:13] + df.drop(columns=["TECHNOLOGY", "MODE_OF_OPERATION"], axis=1, inplace=True) + + df["MONTH"] = pd.Categorical(df["MONTH"], categories=months, ordered=True) + df = df.sort_values(by=["MONTH", "HOUR"]) + df["VALUE"] = df["VALUE"].round(2) + df = df[["YEAR", "MONTH", "HOUR", "NODE_1", "NODE_2", "VALUE"]] + else: + df = pd.DataFrame( + columns=["YEAR", "MONTH", "HOUR", "NODE_1", "NODE_2", "VALUE"] + ) + return df + + +if __name__ == "__main__": + if "snakemake" in globals(): + activity_by_mode_csv = snakemake.input.activity_by_mode + save = snakemake.output.trade_flows + seasons = snakemake.params.seasons + dayparts = snakemake.params.dayparts + timeshift = snakemake.params.timeshift + else: + activity_by_mode_csv = ( + "results/India/results/TotalAnnualTechnologyActivityByMode.csv" + ) + save = "results/India/results/TradeFlows.csv" + seasons = {"S1": [1, 2, 3, 4, 5, 6], "S2": [7, 8, 9, 10, 11, 12]} + dayparts = {"D1": [1, 7], "D2": [7, 13], "D3": [13, 19], "D4": [19, 25]} + timeshift = 0 + + activity_by_mode = pd.read_csv(activity_by_mode_csv, index_col=[0, 1, 2, 3, 4]) + + df = get_trade_flows(activity_by_mode, seasons, dayparts, timeshift) + + df.to_csv(save, index=False) diff --git a/workflow/snakefile b/workflow/snakefile index 4d8330d7..a3edc0a0 100644 --- a/workflow/snakefile +++ b/workflow/snakefile @@ -31,7 +31,6 @@ def get_otoole_data(otoole_config: str, var: str) -> list[str]: "StorageLevelYearStart", "StorageLevelYearFinish", "Trade", - "DiscountedOperatingCost", "TotalDiscountedStorageCost", "ModelPeriodEmissions", "StorageLowerLimit", @@ -86,9 +85,9 @@ rule all: # model results expand('results/{scenario}/result_summaries/{result_summary}.csv', - scenario=config['scenario'], result_summary=result_summaries), + scenario=config['scenario'], result_summary=RESULT_SUMMARIES), expand('results/{scenario}/figures/{result_figure}.html', - scenario=config['scenario'], result_figure = result_figures), + scenario=config['scenario'], result_figure = RESULT_FIGURES), # validation results expand("results/{scenario}/validation/{country}/capacity/{dataset}.png", From e910ba6a3c33fa3fcd1905c000b7d7f705342d17 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 27 Oct 2024 21:25:01 -0700 Subject: [PATCH 04/27] add headline metrics --- workflow/rules/postprocess.smk | 20 ++- .../osemosys_global/summary/headline.py | 140 ++++++++++++++++++ 2 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 workflow/scripts/osemosys_global/summary/headline.py diff --git a/workflow/rules/postprocess.smk b/workflow/rules/postprocess.smk index 13d6af26..5760ed9b 100644 --- a/workflow/rules/postprocess.smk +++ b/workflow/rules/postprocess.smk @@ -13,7 +13,8 @@ RESULT_SUMMARIES = [ "TransmissionCapacity", "PowerCapacity", "GenerationShares", - "NodeCost" + "NodeCost", + "Metrics" ] rule otoole_results: @@ -138,4 +139,19 @@ rule calculate_capacity_by_node: log: log = 'results/{scenario}/logs/generation_shares.log' script: - "../scripts/osemosys_global/summary/capacity.py" \ No newline at end of file + "../scripts/osemosys_global/summary/capacity.py" + +rule calcualte_headline_metrics: + message: + "Calculating Headline Metrics..." + input: + annual_emissions = "results/{scenario}/results/AnnualEmissions.csv", + production_by_technology = "results/{scenario}/results/ProductionByTechnologyAnnual.csv", + total_discounted_cost = "results/{scenario}/results/TotalDiscountedCost.csv", + demand = "results/{scenario}/results/Demand.csv" + output: + metrics = "results/{scenario}/result_summaries/Metrics.csv", + log: + log = 'results/{scenario}/logs/generation_shares.log' + script: + "../scripts/osemosys_global/summary/headline.py" \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/summary/headline.py b/workflow/scripts/osemosys_global/summary/headline.py new file mode 100644 index 00000000..7b659a1a --- /dev/null +++ b/workflow/scripts/osemosys_global/summary/headline.py @@ -0,0 +1,140 @@ +"""Calculates Headline Metrics""" + +import pandas as pd +from constants import RENEWABLES, FOSSIL, CLEAN + + +def get_emissions(annual_emissions: pd.DataFrame) -> pd.DataFrame: + + df = annual_emissions.copy() + + total_emissions = df.VALUE.sum().round(0) + data = ["Emissions", "Million tonnes of CO2-eq.", total_emissions] + return pd.DataFrame([data], columns=["Metric", "Unit", "Value"]) + + +def get_system_cost(total_discounted_cost: pd.DataFrame) -> pd.DataFrame: + + df = total_discounted_cost.copy() + + total_cost = round((df.VALUE.sum() / 1000), 0) + + data = ["Total system cost", "Billion $", total_cost] + return pd.DataFrame([data], columns=["Metric", "Unit", "Value"]) + + +def get_gen_cost( + total_discounted_cost: pd.DataFrame, demand: pd.DataFrame +) -> pd.DataFrame: + + costs = total_discounted_cost.copy() + dem = demand.copy() + + total_cost = costs.VALUE.sum() / 1000 + total_demand = dem.VALUE.sum() + + gen_cost = (total_cost / (total_demand * 0.2778)).round(0) + + data = ["Cost of electricity", "$/MWh", gen_cost] + return pd.DataFrame([data], columns=["Metric", "Unit", "Value"]) + + +def _filter_pwr_techs(production_by_technology: pd.DataFrame) -> pd.DataFrame: + + df = production_by_technology.copy() + + return df[ + (df.index.get_level_values("TECHNOLOGY").str.startswith("PWR")) + & ~(df.index.get_level_values("TECHNOLOGY").str.contains("TRN")) + ] + + +def _filter_rnw_techs(production_by_technology: pd.DataFrame) -> pd.DataFrame: + + df = production_by_technology.copy() + df["tech"] = df.index.get_level_values("TECHNOLOGY").str[0:6] + + rnw_techs = [f"PWR{x}" for x in RENEWABLES] + + df = df[df.tech.isin(rnw_techs)] + + return df["VALUE"].to_frame() + + +def _filter_fossil_techs(production_by_technology: pd.DataFrame) -> pd.DataFrame: + + df = production_by_technology.copy() + df["tech"] = df.index.get_level_values("TECHNOLOGY").str[0:6] + + fossil_techs = [f"PWR{x}" for x in FOSSIL] + + df = df[df.tech.isin(fossil_techs)] + + return df["VALUE"].to_frame() + + +def _filter_clean_techs(production_by_technology: pd.DataFrame) -> pd.DataFrame: + + df = production_by_technology.copy() + df["tech"] = df.index.get_level_values("TECHNOLOGY").str[0:6] + + clean_techs = [f"PWR{x}" for x in CLEAN] + + df = df[df.tech.isin(clean_techs)] + + return df["VALUE"].to_frame() + + +def get_gen_shares(production_by_technology: pd.DataFrame) -> pd.DataFrame: + + gen_total = _filter_pwr_techs(production_by_technology).VALUE.sum() + rnw_total = _filter_rnw_techs(production_by_technology).VALUE.sum() + fsl_total = _filter_fossil_techs(production_by_technology).VALUE.sum() + cln_total = _filter_clean_techs(production_by_technology).VALUE.sum() + + rnw_share = round((rnw_total / gen_total) * 100, 0) + fsl_share = round((fsl_total / gen_total) * 100, 0) + cln_share = round((cln_total / gen_total) * 100, 0) + + data = [ + ["Fossil energy share", "%", fsl_share], + ["Renewable energy share", "%", rnw_share], + ["Clean energy share", "%", cln_share], + ] + + return pd.DataFrame(data, columns=["Metric", "Unit", "Value"]) + + +if __name__ == "__main__": + if "snakemake" in globals(): + annual_emissions_csv = snakemake.input.annual_emissions + production_by_technology_csv = snakemake.input.production_by_technology + total_discounted_cost_csv = snakemake.input.total_discounted_cost + demand_csv = snakemake.input.demand + save = snakemake.output.metrics + else: + annual_emissions_csv = "results/India/results/AnnualEmissions.csv" + production_by_technology_csv = ( + "results/India/results/ProductionByTechnologyAnnual.csv" + ) + total_discounted_cost_csv = "results/India/results/TotalDiscountedCost.csv" + demand_csv = "results/India/results/Demand.csv" + save = "results/India/result_summaries/Metrics.csv" + + annual_emissions = pd.read_csv(annual_emissions_csv, index_col=[0, 1, 2]) + production_by_technology = pd.read_csv( + production_by_technology_csv, index_col=[0, 1, 2, 3] + ) + total_discounted_cost = pd.read_csv(total_discounted_cost_csv, index_col=[0, 1]) + demand = pd.read_csv(demand_csv, index_col=[0, 1, 2, 3]) + + dfs = [] + + dfs.append(get_emissions(annual_emissions)) + dfs.append(get_system_cost(total_discounted_cost)) + dfs.append(get_gen_cost(total_discounted_cost, demand)) + dfs.append(get_gen_shares(production_by_technology)) + + df = pd.concat(dfs) + + df.to_csv(save, index=False) From 5adbb36675e1303dad23dc8c232c789e635f4de7 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 27 Oct 2024 21:52:45 -0700 Subject: [PATCH 05/27] correct units --- workflow/scripts/osemosys_global/summary/headline.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/workflow/scripts/osemosys_global/summary/headline.py b/workflow/scripts/osemosys_global/summary/headline.py index 7b659a1a..2c82e652 100644 --- a/workflow/scripts/osemosys_global/summary/headline.py +++ b/workflow/scripts/osemosys_global/summary/headline.py @@ -30,10 +30,11 @@ def get_gen_cost( costs = total_discounted_cost.copy() dem = demand.copy() - total_cost = costs.VALUE.sum() / 1000 - total_demand = dem.VALUE.sum() + total_cost = costs.VALUE.sum() # $M + total_demand = dem.VALUE.sum() # PJ - gen_cost = (total_cost / (total_demand * 0.2778)).round(0) + # ($M / PJ) ($1000000 / $M) (1 PJ / 1000 TJ) (1 TJ / 1000 GJ) (1 GJ / 1000 MJ) (3600sec / 1000 hr) = 3.6 + gen_cost = ((total_cost / total_demand) * 3.6).round(0) data = ["Cost of electricity", "$/MWh", gen_cost] return pd.DataFrame([data], columns=["Metric", "Unit", "Value"]) From e7954852ffbee390e1f1cc19ee3dab49a474e06f Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Wed, 13 Nov 2024 12:22:09 -0800 Subject: [PATCH 06/27] update otoole req --- setup.cfg | 4 ++-- workflow/envs/osemosys-global.yaml | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/setup.cfg b/setup.cfg index 8b4dad71..19c56d7e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,10 +55,10 @@ install_requires = pytest wbgapi>=1.0.12 scikit-learn>=1.4 - otoole>=1.1.3 + otoole>=1.1.4 scipy dash - geopy + geopy>=2.4.1 [options.packages.find] where = workflow/scripts diff --git a/workflow/envs/osemosys-global.yaml b/workflow/envs/osemosys-global.yaml index 8431fb66..ace64941 100644 --- a/workflow/envs/osemosys-global.yaml +++ b/workflow/envs/osemosys-global.yaml @@ -12,8 +12,5 @@ dependencies: - glpk>=5.0 - pip: - - wbgapi>=1.0.12 - - geopy>=2.4.1 - - otoole==1.1.3 - - dash==2.17 + - otoole>=1.1.4 - ../../. # path to setup.py From 14fafbb2ec0567cc7308d7b68dcbf3bd25af4702 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Wed, 13 Nov 2024 12:59:46 -0800 Subject: [PATCH 07/27] correct otoole config --- resources/otoole.yaml | 25 +++++++++++++++++++++++-- workflow/snakefile | 1 + 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/resources/otoole.yaml b/resources/otoole.yaml index 295483cb..0411b27c 100644 --- a/resources/otoole.yaml +++ b/resources/otoole.yaml @@ -42,7 +42,7 @@ CapitalCostStorage: indices: [REGION,STORAGE,YEAR] type: param dtype: float - default: 2000 + default: 0.001 Conversionld: indices: [TIMESLICE,DAYTYPE] type: param @@ -142,7 +142,7 @@ OperationalLifeStorage: indices: [REGION,STORAGE] type: param dtype: float - default: 10 + default: 1 OutputActivityRatio: indices: [REGION,TECHNOLOGY,FUEL,MODE_OF_OPERATION,YEAR] type: param @@ -332,6 +332,11 @@ AnnualVariableOperatingCost: type: result dtype: float default: 0 +CapitalInvestmentStorage: + indices: [REGION, STORAGE, YEAR] + type: result + dtype: float + default: 0 CapitalInvestment: indices: [REGION, TECHNOLOGY, YEAR] type: result @@ -342,11 +347,22 @@ Demand: type: result dtype: float default: 0 +DiscountedCapitalInvestmentStorage: + short_name: DiscountedCapitalInvestStorage + indices: [REGION, STORAGE, YEAR] + type: result + dtype: float + default: 0 DiscountedCapitalInvestment: indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 +DiscountedCostByStorage: + indices: [REGION, STORAGE, YEAR] + type: result + dtype: float + default: 0 DiscountedCostByTechnology: indices: [REGION, TECHNOLOGY, YEAR] type: result @@ -362,6 +378,11 @@ DiscountedSalvageValue: type: result dtype: float default: 0 +DiscountedSalvageValueStorage: + indices: [REGION, STORAGE, YEAR] + type: result + dtype: float + default: 0 DiscountedTechnologyEmissionsPenalty: short_name: DiscountedTechEmissionsPenalty indices: [REGION, TECHNOLOGY, YEAR] diff --git a/workflow/snakefile b/workflow/snakefile index a3edc0a0..5c6b220b 100644 --- a/workflow/snakefile +++ b/workflow/snakefile @@ -20,6 +20,7 @@ def get_otoole_data(otoole_config: str, var: str) -> list[str]: results = [x for x in otoole if otoole[x]["type"] == var] # no result calcualtions + # or if no result present (ie. no storage runs) missing = [ "NewStorageCapacity", "NumberOfNewTechnologyUnits", From 3ed10ea1d2c0ea86d7d0fa5175fb810cd92c7126 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Wed, 13 Nov 2024 13:21:12 -0800 Subject: [PATCH 08/27] remove storage from gen shares --- .../osemosys_global/summary/gen_shares.py | 19 +++++---- .../osemosys_global/summary/headline.py | 40 +++++-------------- 2 files changed, 22 insertions(+), 37 deletions(-) diff --git a/workflow/scripts/osemosys_global/summary/gen_shares.py b/workflow/scripts/osemosys_global/summary/gen_shares.py index 1750330e..ed9e11e4 100644 --- a/workflow/scripts/osemosys_global/summary/gen_shares.py +++ b/workflow/scripts/osemosys_global/summary/gen_shares.py @@ -4,9 +4,11 @@ from typing import Optional from constants import CLEAN, RENEWABLES, FOSSIL +# do not include storage in generation share calcualtion +EXCLUDE = ["LDS", "SDS"] def _get_gen_by_node( - production: pd.DataFrame, carriers: Optional[list[str]] = None + production: pd.DataFrame, include: Optional[list[str]] = None, exclude: Optional[list[str]] = None ) -> pd.DataFrame: df = production.copy() @@ -16,8 +18,11 @@ def _get_gen_by_node( df["TECH"] = df.index.get_level_values("TECHNOLOGY").str[3:6] df["NODE"] = df.index.get_level_values("TECHNOLOGY").str[6:11] - if carriers: - df = df[df.TECH.isin(carriers)] + if exclude: + df = df[~df.TECH.isin(exclude)].copy() + + if include: + df = df[df.TECH.isin(include)].copy() return ( df.reset_index()[["REGION", "NODE", "YEAR", "VALUE"]] @@ -35,10 +40,10 @@ def calc_generation_shares(production_by_technology: pd.DataFrame) -> pd.DataFra & ~(df.index.get_level_values("TECHNOLOGY").str.contains("TRN")) ] - total = _get_gen_by_node(df).rename(columns={"VALUE": "TOTAL"}) - clean = _get_gen_by_node(df, CLEAN).rename(columns={"VALUE": "CLEAN"}) - renewable = _get_gen_by_node(df, RENEWABLES).rename(columns={"VALUE": "RENEWABLE"}) - fossil = _get_gen_by_node(df, FOSSIL).rename(columns={"VALUE": "FOSSIL"}) + total = _get_gen_by_node(df, exclude=EXCLUDE).rename(columns={"VALUE": "TOTAL"}) + clean = _get_gen_by_node(df, include=CLEAN).rename(columns={"VALUE": "CLEAN"}) + renewable = _get_gen_by_node(df, include=RENEWABLES).rename(columns={"VALUE": "RENEWABLE"}) + fossil = _get_gen_by_node(df, include=FOSSIL).rename(columns={"VALUE": "FOSSIL"}) shares = ( total.join(clean, how="outer") diff --git a/workflow/scripts/osemosys_global/summary/headline.py b/workflow/scripts/osemosys_global/summary/headline.py index 2c82e652..5f93775f 100644 --- a/workflow/scripts/osemosys_global/summary/headline.py +++ b/workflow/scripts/osemosys_global/summary/headline.py @@ -46,42 +46,22 @@ def _filter_pwr_techs(production_by_technology: pd.DataFrame) -> pd.DataFrame: return df[ (df.index.get_level_values("TECHNOLOGY").str.startswith("PWR")) + & ~(df.index.get_level_values("TECHNOLOGY").str.startswith("PWRLDS")) + & ~(df.index.get_level_values("TECHNOLOGY").str.startswith("PWRSDS")) & ~(df.index.get_level_values("TECHNOLOGY").str.contains("TRN")) ] -def _filter_rnw_techs(production_by_technology: pd.DataFrame) -> pd.DataFrame: - - df = production_by_technology.copy() - df["tech"] = df.index.get_level_values("TECHNOLOGY").str[0:6] - - rnw_techs = [f"PWR{x}" for x in RENEWABLES] - - df = df[df.tech.isin(rnw_techs)] - - return df["VALUE"].to_frame() - - -def _filter_fossil_techs(production_by_technology: pd.DataFrame) -> pd.DataFrame: - - df = production_by_technology.copy() - df["tech"] = df.index.get_level_values("TECHNOLOGY").str[0:6] - - fossil_techs = [f"PWR{x}" for x in FOSSIL] - - df = df[df.tech.isin(fossil_techs)] - - return df["VALUE"].to_frame() - - -def _filter_clean_techs(production_by_technology: pd.DataFrame) -> pd.DataFrame: +def _filter_techs( + production_by_technology: pd.DataFrame, carriers: list[str] +) -> pd.DataFrame: df = production_by_technology.copy() df["tech"] = df.index.get_level_values("TECHNOLOGY").str[0:6] - clean_techs = [f"PWR{x}" for x in CLEAN] + techs = [f"PWR{x}" for x in carriers] - df = df[df.tech.isin(clean_techs)] + df = df[df.tech.isin(techs)] return df["VALUE"].to_frame() @@ -89,9 +69,9 @@ def _filter_clean_techs(production_by_technology: pd.DataFrame) -> pd.DataFrame: def get_gen_shares(production_by_technology: pd.DataFrame) -> pd.DataFrame: gen_total = _filter_pwr_techs(production_by_technology).VALUE.sum() - rnw_total = _filter_rnw_techs(production_by_technology).VALUE.sum() - fsl_total = _filter_fossil_techs(production_by_technology).VALUE.sum() - cln_total = _filter_clean_techs(production_by_technology).VALUE.sum() + rnw_total = _filter_techs(production_by_technology, RENEWABLES).VALUE.sum() + fsl_total = _filter_techs(production_by_technology, FOSSIL).VALUE.sum() + cln_total = _filter_techs(production_by_technology, CLEAN).VALUE.sum() rnw_share = round((rnw_total / gen_total) * 100, 0) fsl_share = round((fsl_total / gen_total) * 100, 0) From 5b2f89e353b0d89c581ebdd686a0257517d84326 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Wed, 13 Nov 2024 13:43:29 -0800 Subject: [PATCH 09/27] update node cost --- .../osemosys_global/summary/node_cost.py | 80 ++++++++++++++----- 1 file changed, 59 insertions(+), 21 deletions(-) diff --git a/workflow/scripts/osemosys_global/summary/node_cost.py b/workflow/scripts/osemosys_global/summary/node_cost.py index 7244a759..07e0e2b6 100644 --- a/workflow/scripts/osemosys_global/summary/node_cost.py +++ b/workflow/scripts/osemosys_global/summary/node_cost.py @@ -1,42 +1,64 @@ """Gets power generation cost per node""" import pandas as pd +from pathlib import Path -def get_cost_per_node(discounted_cost_tech: pd.DataFrame) -> pd.DataFrame: + +def get_tech_cost_per_node(discounted_cost_tech: pd.DataFrame) -> pd.DataFrame: """Only of power generation technologies""" df = discounted_cost_tech.copy() - + df = df[ (df.index.get_level_values("TECHNOLOGY").str.startswith("PWR")) & ~(df.index.get_level_values("TECHNOLOGY").str.contains("TRN")) ] - + df["NODE"] = df.index.get_level_values("TECHNOLOGY").str[6:11] - - return df.reset_index()[["REGION", "NODE", "YEAR", "VALUE"]].groupby(["REGION", "NODE", "YEAR"]).sum() - + + return ( + df.reset_index()[["REGION", "NODE", "YEAR", "VALUE"]] + .groupby(["REGION", "NODE", "YEAR"]) + .sum() + ) + +def get_storage_cost_per_node(discounted_cost_storage: pd.DataFrame) -> pd.DataFrame: + + df = discounted_cost_storage.copy() + df["NODE"] = df.index.get_level_values("STORAGE").str[3:8] + return ( + df.reset_index()[["REGION", "NODE", "YEAR", "VALUE"]] + .groupby(["REGION", "NODE", "YEAR"]) + .sum() + ) + def get_demand_per_node(demand: pd.DataFrame) -> pd.DataFrame: - + df = demand.copy() - + df = df[df.index.get_level_values("FUEL").str.startswith("ELC")] df["NODE"] = df.index.get_level_values("FUEL").str[3:8] - - return df.reset_index()[["REGION", "NODE", "YEAR", "VALUE"]].groupby(["REGION", "NODE", "YEAR"]).sum() - - + + return ( + df.reset_index()[["REGION", "NODE", "YEAR", "VALUE"]] + .groupby(["REGION", "NODE", "YEAR"]) + .sum() + ) + + def get_pwr_cost_per_node(demand: pd.DataFrame, cost: pd.DataFrame) -> pd.DataFrame: """Gets power generation cost per node in $/MWh""" - + # ($M / PJ) (1PJ / 1000 TJ) (1TJ / 1000 GJ) (1GJ / 1000 MJ) ($1000000 / $M) (3600sec / hr) df = cost.div(demand) return df.mul(3.6) - - + + if __name__ == "__main__": if "snakemake" in globals(): - discounted_cost_by_technology_csv = snakemake.input.discounted_cost_by_technology + discounted_cost_by_technology_csv = ( + snakemake.input.discounted_cost_by_technology + ) demand_csv = snakemake.input.demand save = snakemake.output.node_cost else: @@ -46,14 +68,30 @@ def get_pwr_cost_per_node(demand: pd.DataFrame, cost: pd.DataFrame) -> pd.DataFr demand_csv = "results/India/results/Demand.csv" save = "results/India/results/NodeCost.csv" - discounted_cost_by_technology_raw = pd.read_csv( + discounted_cost_by_technology = pd.read_csv( discounted_cost_by_technology_csv, index_col=[0, 1, 2] ) - demand_raw = pd.read_csv( - demand_csv, index_col=[0, 1, 2, 3] - ) + demand_raw = pd.read_csv(demand_csv, index_col=[0, 1, 2, 3]) - cost = get_cost_per_node(discounted_cost_by_technology_raw) + # handle the storage discounted costs outside of snakemake + # if running a model without storage, its not guaranteed that result + # csvs will be generated with all solvers + try: + parent = Path(discounted_cost_by_technology_csv).parent + discounted_cost_by_storage_csv = Path(parent, "DiscountedCostByStorage.csv") + discounted_cost_by_storage = pd.read_csv( + discounted_cost_by_storage_csv, index_col=[0, 1, 2] + ) + except FileExistsError: + discounted_cost_by_storage_raw = pd.DataFrame( + columns=["REGION", "STORAGE", "YEAR", "VALUE"] + ).set_index(["REGION", "STORAGE", "YEAR"]) + + tech_cost = get_tech_cost_per_node(discounted_cost_by_technology) + storage_cost = get_storage_cost_per_node(discounted_cost_by_storage) + + cost = tech_cost.add(storage_cost, fill_value=0) + demand = get_demand_per_node(demand_raw) df = get_pwr_cost_per_node(demand, cost) From f4d04255afeed51b2557d27c3309f2b55e3cb3eb Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Wed, 13 Nov 2024 13:51:48 -0800 Subject: [PATCH 10/27] correct otoole config --- resources/otoole.yaml | 56 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/resources/otoole.yaml b/resources/otoole.yaml index 0411b27c..dd44716e 100644 --- a/resources/otoole.yaml +++ b/resources/otoole.yaml @@ -322,6 +322,12 @@ AnnualTechnologyEmission: type: result dtype: float default: 0 +AnnualTechnologyEmissionsPenalty: + indices: [REGION, TECHNOLOGY, YEAR] + type: result + dtype: float + default: 0 + short_name: AnnualTechEmissionsPenalty AnnualTechnologyEmissionByMode: indices: [REGION, TECHNOLOGY, EMISSION, MODE_OF_OPERATION, YEAR] type: result @@ -332,13 +338,13 @@ AnnualVariableOperatingCost: type: result dtype: float default: 0 -CapitalInvestmentStorage: - indices: [REGION, STORAGE, YEAR] +CapitalInvestment: + indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 -CapitalInvestment: - indices: [REGION, TECHNOLOGY, YEAR] +CapitalInvestmentStorage: + indices: [REGION, STORAGE, YEAR] type: result dtype: float default: 0 @@ -347,14 +353,14 @@ Demand: type: result dtype: float default: 0 -DiscountedCapitalInvestmentStorage: - short_name: DiscountedCapitalInvestStorage - indices: [REGION, STORAGE, YEAR] +DiscountedCapitalInvestment: + indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float default: 0 -DiscountedCapitalInvestment: - indices: [REGION, TECHNOLOGY, YEAR] +DiscountedCapitalInvestmentStorage: + short_name: DiscountedCapitalInvestStorage + indices: [REGION, STORAGE, YEAR] type: result dtype: float default: 0 @@ -368,7 +374,7 @@ DiscountedCostByTechnology: type: result dtype: float default: 0 -DiscountedOperationalCost: +DiscountedOperatingCost: indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float @@ -389,6 +395,21 @@ DiscountedTechnologyEmissionsPenalty: type: result dtype: float default: 0 +ModelPeriodEmissions: + indices: [REGION, EMISSION] + type: result + dtype: float + default: 0 +NetChargeWithinDay: + indices: [REGION, STORAGE, SEASON, DAYTYPE, DAILYTIMEBRACKET, YEAR] + type: result + dtype: float + default: 0 +NetChargeWithinYear: + indices: [REGION, STORAGE, SEASON, DAYTYPE, DAILYTIMEBRACKET, YEAR] + type: result + dtype: float + default: 0 NewCapacity: indices: [REGION, TECHNOLOGY, YEAR] type: result @@ -404,6 +425,11 @@ NumberOfNewTechnologyUnits: type: result dtype: float default: 0 +OperatingCost: + indices: [REGION, TECHNOLOGY, YEAR] + type: result + dtype: float + default: 0 ProductionByTechnology: indices: [REGION, TIMESLICE, TECHNOLOGY, FUEL, YEAR] type: result @@ -475,6 +501,16 @@ StorageLevelYearFinish: type: result dtype: float default: 0 +StorageLowerLimit: + indices: [REGION, STORAGE, YEAR] + type: result + dtype: float + default: 0 +StorageUpperLimit: + indices: [REGION, STORAGE, YEAR] + type: result + dtype: float + default: 0 TotalAnnualTechnologyActivityByMode: short_name: TotalAnnualTechActivityByMode indices: [REGION, TECHNOLOGY, MODE_OF_OPERATION, YEAR] From 4c2242f7ed5908fbba6348498eb5a7beb77308fb Mon Sep 17 00:00:00 2001 From: maartenbrinkerink <65602545+maartenbrinkerink@users.noreply.github.com> Date: Thu, 14 Nov 2024 09:10:35 -0500 Subject: [PATCH 11/27] Added DiscountedOperatingCost to missing files as workflow wouldn't run --- workflow/snakefile | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow/snakefile b/workflow/snakefile index 5c6b220b..62f00124 100644 --- a/workflow/snakefile +++ b/workflow/snakefile @@ -32,6 +32,7 @@ def get_otoole_data(otoole_config: str, var: str) -> list[str]: "StorageLevelYearStart", "StorageLevelYearFinish", "Trade", + "DiscountedOperatingCost", "TotalDiscountedStorageCost", "ModelPeriodEmissions", "StorageLowerLimit", From 38359f71cfc8a4c636ff3b31fda9e1d0f0330d13 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 17 Nov 2024 20:33:44 -0800 Subject: [PATCH 12/27] add renewables to summaries --- .../osemosys_global/summary/constants.py | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/workflow/scripts/osemosys_global/summary/constants.py b/workflow/scripts/osemosys_global/summary/constants.py index 5b5bc37b..9f4b1f1d 100644 --- a/workflow/scripts/osemosys_global/summary/constants.py +++ b/workflow/scripts/osemosys_global/summary/constants.py @@ -1,35 +1,35 @@ """Constants for summary statistics""" -RENEWABLES = ["GEO", "HYD", "SPV", "CSP", "WAS", "WAV", "WON", "WOF"] +RENEWABLES = ["BIO", "GEO", "HYD", "SPV", "CSP", "WAS", "WAV", "WON", "WOF"] CLEAN = ["BIO", "GEO", "HYD", "SPV", "CSP", "WAS", "WAV", "WON", "WOF", "URN"] FOSSIL = ["CCG", "COA", "COG", "OCG", "OIL", "OTH", "PET"] MONTH_NAMES = { - 1: 'Jan', - 2: 'Feb', - 3: 'Mar', - 4: 'Apr', - 5: 'May', - 6: 'Jun', - 7: 'Jul', - 8: 'Aug', - 9: 'Sep', - 10: 'Oct', - 11: 'Nov', - 12: 'Dec', + 1: "Jan", + 2: "Feb", + 3: "Mar", + 4: "Apr", + 5: "May", + 6: "Jun", + 7: "Jul", + 8: "Aug", + 9: "Sep", + 10: "Oct", + 11: "Nov", + 12: "Dec", } DAYS_PER_MONTH = { - 'Jan': 31, - 'Feb': 28, - 'Mar': 31, - 'Apr': 30, - 'May': 31, - 'Jun': 30, - 'Jul': 31, - 'Aug': 31, - 'Sep': 30, - 'Oct': 31, - 'Nov': 30, - 'Dec': 31, -} \ No newline at end of file + "Jan": 31, + "Feb": 28, + "Mar": 31, + "Apr": 30, + "May": 31, + "Jun": 30, + "Jul": 31, + "Aug": 31, + "Sep": 30, + "Oct": 31, + "Nov": 30, + "Dec": 31, +} From f3c7cd90d62f139bdaf930fc7ce8b80d6c314bfb Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 17 Nov 2024 20:45:26 -0800 Subject: [PATCH 13/27] dynamic storage names for gen shares --- workflow/rules/postprocess.smk | 2 ++ .../osemosys_global/summary/gen_shares.py | 25 +++++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/workflow/rules/postprocess.smk b/workflow/rules/postprocess.smk index 5760ed9b..4edd238f 100644 --- a/workflow/rules/postprocess.smk +++ b/workflow/rules/postprocess.smk @@ -119,6 +119,8 @@ rule calculate_cost_by_node: rule calculate_generation_shares: message: "Calculating Generaion Fuel Shares..." + params: + storage = config['storage_parameters'], input: production_by_technology = "results/{scenario}/results/ProductionByTechnology.csv", output: diff --git a/workflow/scripts/osemosys_global/summary/gen_shares.py b/workflow/scripts/osemosys_global/summary/gen_shares.py index ed9e11e4..5ed82777 100644 --- a/workflow/scripts/osemosys_global/summary/gen_shares.py +++ b/workflow/scripts/osemosys_global/summary/gen_shares.py @@ -4,11 +4,11 @@ from typing import Optional from constants import CLEAN, RENEWABLES, FOSSIL -# do not include storage in generation share calcualtion -EXCLUDE = ["LDS", "SDS"] def _get_gen_by_node( - production: pd.DataFrame, include: Optional[list[str]] = None, exclude: Optional[list[str]] = None + production: pd.DataFrame, + include: Optional[list[str]] = None, + exclude: Optional[list[str]] = None, ) -> pd.DataFrame: df = production.copy() @@ -31,7 +31,12 @@ def _get_gen_by_node( ) -def calc_generation_shares(production_by_technology: pd.DataFrame) -> pd.DataFrame: +def calc_generation_shares( + production_by_technology: pd.DataFrame, exclusions: Optional[list[str]] = None +) -> pd.DataFrame: + + if not exclusions: + exclusions = [] df = production_by_technology.copy() @@ -40,9 +45,11 @@ def calc_generation_shares(production_by_technology: pd.DataFrame) -> pd.DataFra & ~(df.index.get_level_values("TECHNOLOGY").str.contains("TRN")) ] - total = _get_gen_by_node(df, exclude=EXCLUDE).rename(columns={"VALUE": "TOTAL"}) + total = _get_gen_by_node(df, exclude=exclusions).rename(columns={"VALUE": "TOTAL"}) clean = _get_gen_by_node(df, include=CLEAN).rename(columns={"VALUE": "CLEAN"}) - renewable = _get_gen_by_node(df, include=RENEWABLES).rename(columns={"VALUE": "RENEWABLE"}) + renewable = _get_gen_by_node(df, include=RENEWABLES).rename( + columns={"VALUE": "RENEWABLE"} + ) fossil = _get_gen_by_node(df, include=FOSSIL).rename(columns={"VALUE": "FOSSIL"}) shares = ( @@ -62,17 +69,21 @@ def calc_generation_shares(production_by_technology: pd.DataFrame) -> pd.DataFra if __name__ == "__main__": if "snakemake" in globals(): production_by_technology_annual_csv = snakemake.input.production_by_technology + storage = snakemake.params.storage save = snakemake.output.generation_shares else: production_by_technology_annual_csv = ( "results/India/results/ProductionByTechnologyAnnual.csv" ) + storage = {"SDS": [], "LDS": []} save = "results/India/results/GenerationShares.csv" production_by_technology_annual = pd.read_csv( production_by_technology_annual_csv, index_col=[0, 1, 2, 3] ) - df = calc_generation_shares(production_by_technology_annual) + exclusions = list(storage) + + df = calc_generation_shares(production_by_technology_annual, exclusions) df.to_csv(save, index=True) From f47919ded2fc3886f7a557e2c73b7dc8203a7b70 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 17 Nov 2024 21:01:27 -0800 Subject: [PATCH 14/27] dynamic exclusions for headline metrics --- workflow/rules/postprocess.smk | 20 ++--------- .../osemosys_global/summary/headline.py | 35 ++++++++++++++----- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/workflow/rules/postprocess.smk b/workflow/rules/postprocess.smk index 4edd238f..5c0eabb3 100644 --- a/workflow/rules/postprocess.smk +++ b/workflow/rules/postprocess.smk @@ -56,24 +56,6 @@ rule visualisation: shell: 'python workflow/scripts/osemosys_global/visualise.py {params.input_data} {params.result_data} {params.scenario_figs_dir} {params.cost_line_expansion_xlsx} {params.countries} {params.results_by_country} {params.years} 2> {log}' -# rule summarise_results: -# message: -# 'Generating summary of results...' -# input: -# csv_files = expand('results/{{scenario}}/results/{result_file}.csv', result_file = OTOOLE_RESULTS), -# params: -# start_year = config['startYear'], -# end_year = config['endYear'], -# dayparts = config['dayparts'], -# seasons = config['seasons'], -# output: -# expand('results/{{scenario}}/result_summaries/{result_summary}.csv', -# result_summary = result_summaries), -# log: -# log = 'results/{scenario}/logs/summarise_results.log' -# shell: -# 'python workflow/scripts/osemosys_global/summarise_results.py 2> {log}' - rule calculate_trade_flows: message: "Calculating Hourly Trade Flows" @@ -146,6 +128,8 @@ rule calculate_capacity_by_node: rule calcualte_headline_metrics: message: "Calculating Headline Metrics..." + params: + storage = config['storage_parameters'], input: annual_emissions = "results/{scenario}/results/AnnualEmissions.csv", production_by_technology = "results/{scenario}/results/ProductionByTechnologyAnnual.csv", diff --git a/workflow/scripts/osemosys_global/summary/headline.py b/workflow/scripts/osemosys_global/summary/headline.py index 5f93775f..e5995e20 100644 --- a/workflow/scripts/osemosys_global/summary/headline.py +++ b/workflow/scripts/osemosys_global/summary/headline.py @@ -1,6 +1,7 @@ """Calculates Headline Metrics""" import pandas as pd +from typing import Optional from constants import RENEWABLES, FOSSIL, CLEAN @@ -40,16 +41,29 @@ def get_gen_cost( return pd.DataFrame([data], columns=["Metric", "Unit", "Value"]) -def _filter_pwr_techs(production_by_technology: pd.DataFrame) -> pd.DataFrame: +def _filter_pwr_techs( + production_by_technology: pd.DataFrame, exclusions: Optional[list[str]] +) -> pd.DataFrame: + """Filters for only power techs + + exclusions: Optional[list[str]] + Additional 'PWR' techs to filter for. For example, if ['LDS' and 'SDS'] are provided, + will filter out 'PWRLDS' and 'PWRSDS' values + """ df = production_by_technology.copy() - return df[ + df = df[ (df.index.get_level_values("TECHNOLOGY").str.startswith("PWR")) - & ~(df.index.get_level_values("TECHNOLOGY").str.startswith("PWRLDS")) - & ~(df.index.get_level_values("TECHNOLOGY").str.startswith("PWRSDS")) & ~(df.index.get_level_values("TECHNOLOGY").str.contains("TRN")) - ] + ].copy() + + if not exclusions: + return df + + prefixes = tuple([f"PWR{x}" for x in exclusions]) + + return df[~df.index.get_level_values("TECHNOLOGY").str.startswith(prefixes)].copy() def _filter_techs( @@ -66,9 +80,11 @@ def _filter_techs( return df["VALUE"].to_frame() -def get_gen_shares(production_by_technology: pd.DataFrame) -> pd.DataFrame: +def get_gen_shares( + production_by_technology: pd.DataFrame, exclusions: Optional[list[str]] = None +) -> pd.DataFrame: - gen_total = _filter_pwr_techs(production_by_technology).VALUE.sum() + gen_total = _filter_pwr_techs(production_by_technology, exclusions).VALUE.sum() rnw_total = _filter_techs(production_by_technology, RENEWABLES).VALUE.sum() fsl_total = _filter_techs(production_by_technology, FOSSIL).VALUE.sum() cln_total = _filter_techs(production_by_technology, CLEAN).VALUE.sum() @@ -88,12 +104,14 @@ def get_gen_shares(production_by_technology: pd.DataFrame) -> pd.DataFrame: if __name__ == "__main__": if "snakemake" in globals(): + storage = snakemake.params.storage annual_emissions_csv = snakemake.input.annual_emissions production_by_technology_csv = snakemake.input.production_by_technology total_discounted_cost_csv = snakemake.input.total_discounted_cost demand_csv = snakemake.input.demand save = snakemake.output.metrics else: + storage = {"SDS": [], "LDS": []} annual_emissions_csv = "results/India/results/AnnualEmissions.csv" production_by_technology_csv = ( "results/India/results/ProductionByTechnologyAnnual.csv" @@ -108,13 +126,14 @@ def get_gen_shares(production_by_technology: pd.DataFrame) -> pd.DataFrame: ) total_discounted_cost = pd.read_csv(total_discounted_cost_csv, index_col=[0, 1]) demand = pd.read_csv(demand_csv, index_col=[0, 1, 2, 3]) + exclusions = list(storage) dfs = [] dfs.append(get_emissions(annual_emissions)) dfs.append(get_system_cost(total_discounted_cost)) dfs.append(get_gen_cost(total_discounted_cost, demand)) - dfs.append(get_gen_shares(production_by_technology)) + dfs.append(get_gen_shares(production_by_technology, exclusions)) df = pd.concat(dfs) From f0e1e14ead500d9fe26b1b9205d1eb4a81536a3a Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 17 Nov 2024 21:12:11 -0800 Subject: [PATCH 15/27] change emission intensity to use demand --- workflow/rules/postprocess.smk | 2 +- .../summary/carbon_intensity.py | 35 ++++++++----------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/workflow/rules/postprocess.smk b/workflow/rules/postprocess.smk index 5c0eabb3..ade838a8 100644 --- a/workflow/rules/postprocess.smk +++ b/workflow/rules/postprocess.smk @@ -76,7 +76,7 @@ rule calculate_carbon_intensity: message: "Calculating Carbon Intensity..." input: - production_by_technology = "results/{scenario}/results/ProductionByTechnologyAnnual.csv", + demand = "results/{scenario}/results/Demand.csv", annual_emissions = "results/{scenario}/results/AnnualEmissions.csv", output: emission_intensity = "results/{scenario}/result_summaries/AnnualEmissionIntensity.csv", diff --git a/workflow/scripts/osemosys_global/summary/carbon_intensity.py b/workflow/scripts/osemosys_global/summary/carbon_intensity.py index 5456ce80..af95c629 100644 --- a/workflow/scripts/osemosys_global/summary/carbon_intensity.py +++ b/workflow/scripts/osemosys_global/summary/carbon_intensity.py @@ -7,15 +7,12 @@ import pandas as pd -def format_production(production_by_technology_annual: pd.DataFrame) -> pd.DataFrame: +def format_demand(demand: pd.DataFrame) -> pd.DataFrame: - df = production_by_technology_annual.copy() + df = demand.copy() - df = df[df.TECHNOLOGY.str.startswith("PWR")] - df["TECH"] = df.TECHNOLOGY.str[3:6] - df["CTRY"] = df.TECHNOLOGY.str[6:9] - df = df[~(df.TECH.isin(["BAT", "TRN"]))] - df = df.drop(columns=["TECHNOLOGY", "FUEL", "TECH"]) + df["CTRY"] = df.FUEL.str[3:6] + df = df.drop(columns=["TIMESLICE", "FUEL"]) return df.groupby(["REGION", "CTRY", "YEAR"]).sum() @@ -28,47 +25,43 @@ def format_emissions(annual_emissions: pd.DataFrame) -> pd.DataFrame: def calculate_emission_intensity( - production: pd.DataFrame, emissions: pd.DataFrame + demand: pd.DataFrame, emissions: pd.DataFrame ) -> pd.DataFrame: - p = production.copy().rename(columns={"VALUE": "PRODUCTION_PJ"}) + d = demand.copy().rename(columns={"VALUE": "DEMAND_PJ"}) e = emissions.copy().rename(columns={"VALUE": "EMISSIONS_MT"}) - df = p.join(e).fillna(0) + df = d.join(e).fillna(0) df = df.droplevel("CTRY") # emission retains country # 1 PJ (1000 TJ / PJ) (1000 GJ / TJ) (1000 MJ / GJ) (1000 kJ / MJ) (hr / 3600sec) - df["PRODUCTION_kwh"] = ( - df.PRODUCTION_PJ.mul(1000).mul(1000).mul(1000).mul(1000).div(3600) - ) + df["DEMAND_kwh"] = df.DEMAND_PJ.mul(1000).mul(1000).mul(1000).mul(1000).div(3600) # 1 MT (1,000,000 T / MT) (1000 kg / T) (1000g / kg) df["EMISSIONS_g"] = df.EMISSIONS_MT.mul(1000000).mul(1000).mul(1000) # intensity in g/kwh - df["VALUE"] = df.EMISSIONS_g.div(df.PRODUCTION_kwh) + df["VALUE"] = df.EMISSIONS_g.div(df.DEMAND_kwh) return df["VALUE"].to_frame() if __name__ == "__main__": if "snakemake" in globals(): - production_by_technology_annual_csv = snakemake.input.production_by_technology + demand_csv = snakemake.input.demand annual_emissions_csv = snakemake.input.annual_emissions save = snakemake.output.emission_intensity else: - production_by_technology_annual_csv = ( - "results/India/results/ProductionByTechnologyAnnual.csv" - ) + demand_csv = "results/India/results/Demand.csv" annual_emissions_csv = "results/India/results/AnnualEmissions.csv" save = "results/India/results/AnnualEmissionIntensity.csv" - production_by_technology_annual = pd.read_csv(production_by_technology_annual_csv) + demand = pd.read_csv(demand_csv) annual_emissions = pd.read_csv(annual_emissions_csv) - prod = format_production(production_by_technology_annual) + demand = format_demand(demand) emissions = format_emissions(annual_emissions) - emission_intensity = calculate_emission_intensity(prod, emissions).round(2) + emission_intensity = calculate_emission_intensity(demand, emissions).round(2) emission_intensity.to_csv(save, index=True) From 3ffa2cdb35c50a7c59bc334780eeb12ad61dd1cb Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 17 Nov 2024 21:23:54 -0800 Subject: [PATCH 16/27] gen shares by country --- workflow/rules/postprocess.smk | 3 +- .../osemosys_global/summary/gen_shares.py | 81 +++++++++++++++++-- 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/workflow/rules/postprocess.smk b/workflow/rules/postprocess.smk index ade838a8..685ed048 100644 --- a/workflow/rules/postprocess.smk +++ b/workflow/rules/postprocess.smk @@ -106,7 +106,8 @@ rule calculate_generation_shares: input: production_by_technology = "results/{scenario}/results/ProductionByTechnology.csv", output: - generation_shares = "results/{scenario}/result_summaries/GenerationShares.csv", + generation_shares_node = "results/{scenario}/result_summaries/GenerationSharesNode.csv", + generation_shares_country = "results/{scenario}/result_summaries/GenerationSharesCountry.csv", log: log = 'results/{scenario}/logs/generation_shares.log' script: diff --git a/workflow/scripts/osemosys_global/summary/gen_shares.py b/workflow/scripts/osemosys_global/summary/gen_shares.py index 5ed82777..6c5093f1 100644 --- a/workflow/scripts/osemosys_global/summary/gen_shares.py +++ b/workflow/scripts/osemosys_global/summary/gen_shares.py @@ -31,7 +31,33 @@ def _get_gen_by_node( ) -def calc_generation_shares( +def _get_gen_by_country( + production: pd.DataFrame, + include: Optional[list[str]] = None, + exclude: Optional[list[str]] = None, +) -> pd.DataFrame: + + df = production.copy() + + assert "TECHNOLOGY" in df.index.names + + df["TECH"] = df.index.get_level_values("TECHNOLOGY").str[3:6] + df["COUNTRY"] = df.index.get_level_values("TECHNOLOGY").str[6:9] + + if exclude: + df = df[~df.TECH.isin(exclude)].copy() + + if include: + df = df[df.TECH.isin(include)].copy() + + return ( + df.reset_index()[["REGION", "COUNTRY", "YEAR", "VALUE"]] + .groupby(["REGION", "COUNTRY", "YEAR"]) + .sum() + ) + + +def calc_generation_shares_node( production_by_technology: pd.DataFrame, exclusions: Optional[list[str]] = None ) -> pd.DataFrame: @@ -66,17 +92,58 @@ def calc_generation_shares( return shares[["CLEAN", "RENEWABLE", "FOSSIL"]].round(1) +def calc_generation_shares_country( + production_by_technology: pd.DataFrame, exclusions: Optional[list[str]] = None +) -> pd.DataFrame: + + if not exclusions: + exclusions = [] + + df = production_by_technology.copy() + + df = df[ + (df.index.get_level_values("TECHNOLOGY").str.startswith("PWR")) + & ~(df.index.get_level_values("TECHNOLOGY").str.contains("TRN")) + ] + + total = _get_gen_by_country(df, exclude=exclusions).rename( + columns={"VALUE": "TOTAL"} + ) + clean = _get_gen_by_country(df, include=CLEAN).rename(columns={"VALUE": "CLEAN"}) + renewable = _get_gen_by_country(df, include=RENEWABLES).rename( + columns={"VALUE": "RENEWABLE"} + ) + fossil = _get_gen_by_country(df, include=FOSSIL).rename(columns={"VALUE": "FOSSIL"}) + + shares = ( + total.join(clean, how="outer") + .join(renewable, how="outer") + .join(fossil, how="outer") + .fillna(0) + ) + + shares["CLEAN"] = shares.CLEAN.div(shares.TOTAL).mul(100) + shares["RENEWABLE"] = shares.RENEWABLE.div(shares.TOTAL).mul(100) + shares["FOSSIL"] = shares.FOSSIL.div(shares.TOTAL).mul(100) + + return shares[["CLEAN", "RENEWABLE", "FOSSIL"]].round(1) + + if __name__ == "__main__": if "snakemake" in globals(): production_by_technology_annual_csv = snakemake.input.production_by_technology storage = snakemake.params.storage - save = snakemake.output.generation_shares + gen_shares_node = snakemake.output.generation_shares_node + gen_shares_country = snakemake.output.generation_shares_country else: production_by_technology_annual_csv = ( "results/India/results/ProductionByTechnologyAnnual.csv" ) storage = {"SDS": [], "LDS": []} - save = "results/India/results/GenerationShares.csv" + gen_shares_node = "results/India/result_summaries/GenerationSharesNode.csv" + gen_shares_country = ( + "results/India/result_summaries/GenerationSharesCountry.csv" + ) production_by_technology_annual = pd.read_csv( production_by_technology_annual_csv, index_col=[0, 1, 2, 3] @@ -84,6 +151,10 @@ def calc_generation_shares( exclusions = list(storage) - df = calc_generation_shares(production_by_technology_annual, exclusions) + nodes = calc_generation_shares_node(production_by_technology_annual, exclusions) + country = calc_generation_shares_country( + production_by_technology_annual, exclusions + ) - df.to_csv(save, index=True) + nodes.to_csv(gen_shares_node, index=True) + country.to_csv(gen_shares_country, index=True) From 7f00217421a09c59c0c5df2051390150ca64dce1 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 17 Nov 2024 21:37:11 -0800 Subject: [PATCH 17/27] capacities at country levels --- workflow/rules/postprocess.smk | 8 +- .../osemosys_global/summary/capacity.py | 77 +++++++++++++------ 2 files changed, 59 insertions(+), 26 deletions(-) diff --git a/workflow/rules/postprocess.smk b/workflow/rules/postprocess.smk index 685ed048..b50b1456 100644 --- a/workflow/rules/postprocess.smk +++ b/workflow/rules/postprocess.smk @@ -113,14 +113,16 @@ rule calculate_generation_shares: script: "../scripts/osemosys_global/summary/gen_shares.py" -rule calculate_capacity_by_node: +rule calculate_capacity: message: "Calculating Capacity by Node..." input: total_capacity = "results/{scenario}/results/TotalCapacityAnnual.csv", output: - power_capacity = "results/{scenario}/result_summaries/PowerCapacity.csv", - transmission_capacity = "results/{scenario}/result_summaries/TransmissionCapacity.csv", + power_capacity_node = "results/{scenario}/result_summaries/PowerCapacityNode.csv", + transmission_capacity_node = "results/{scenario}/result_summaries/TransmissionCapacityNode.csv", + power_capacity_country = "results/{scenario}/result_summaries/PowerCapacityCountry.csv", + transmission_capacity_country = "results/{scenario}/result_summaries/TransmissionCapacityCountry.csv", log: log = 'results/{scenario}/logs/generation_shares.log' script: diff --git a/workflow/scripts/osemosys_global/summary/capacity.py b/workflow/scripts/osemosys_global/summary/capacity.py index a73849cd..796fe824 100644 --- a/workflow/scripts/osemosys_global/summary/capacity.py +++ b/workflow/scripts/osemosys_global/summary/capacity.py @@ -2,17 +2,32 @@ import pandas as pd -def calc_trn_capacity(total_capacity_annual: pd.DataFrame) -> pd.DataFrame: + +def calc_trn_capacity( + total_capacity_annual: pd.DataFrame, country: bool +) -> pd.DataFrame: df = total_capacity_annual.copy() df = df[df.index.get_level_values("TECHNOLOGY").str.startswith("TRN")] - df["FROM"] = df.index.get_level_values("TECHNOLOGY").str[3:8] - df["TO"] = df.index.get_level_values("TECHNOLOGY").str[8:13] - return df.reset_index()[["FROM", "TO", "YEAR", "VALUE"]].groupby(["FROM", "TO", "YEAR"]).sum() + if country: + df["FROM"] = df.index.get_level_values("TECHNOLOGY").str[3:6] + df["TO"] = df.index.get_level_values("TECHNOLOGY").str[8:11] + else: + df["FROM"] = df.index.get_level_values("TECHNOLOGY").str[3:8] + df["TO"] = df.index.get_level_values("TECHNOLOGY").str[8:13] + + return ( + df.reset_index()[["FROM", "TO", "YEAR", "VALUE"]] + .groupby(["FROM", "TO", "YEAR"]) + .sum() + ) + -def calc_pwr_capacity(total_capacity_annual: pd.DataFrame) -> pd.DataFrame: +def calc_pwr_capacity( + total_capacity_annual: pd.DataFrame, country: bool +) -> pd.DataFrame: df = total_capacity_annual.copy() @@ -20,30 +35,46 @@ def calc_pwr_capacity(total_capacity_annual: pd.DataFrame) -> pd.DataFrame: (df.index.get_level_values("TECHNOLOGY").str.startswith("PWR")) & ~(df.index.get_level_values("TECHNOLOGY").str.contains("TRN")) ] - + df["TECH"] = df.index.get_level_values("TECHNOLOGY").str[3:6] - df["NODE"] = df.index.get_level_values("TECHNOLOGY").str[6:11] - return df.reset_index()[["TECH", "NODE", "YEAR", "VALUE"]].groupby(["TECH", "NODE", "YEAR"]).sum().rename(columns={"TECH": "TECHNOLOGY"}) + if country: + r = "COUNTRY" + df[r] = df.index.get_level_values("TECHNOLOGY").str[6:9] + else: + r = "NODE" + df[r] = df.index.get_level_values("TECHNOLOGY").str[6:11] + + return ( + df.reset_index()[["TECH", r, "YEAR", "VALUE"]] + .groupby(["TECH", r, "YEAR"]) + .sum() + .rename(columns={"TECH": "TECHNOLOGY"}) + ) + if __name__ == "__main__": if "snakemake" in globals(): total_capacity_csv = snakemake.input.total_capacity - pwr_save = snakemake.output.power_capacity - trn_save = snakemake.output.transmission_capacity + pwr_node_save = snakemake.output.power_capacity_node + trn_node_save = snakemake.output.transmission_capacity_node + pwr_country_save = snakemake.output.power_capacity_country + trn_country_save = snakemake.output.transmission_capacity_country else: - total_capacity_csv = ( - "results/India/results/TotalCapacityAnnual.csv" - ) - pwr_save = "" - trn_save = "" - - total_capacity_annual = pd.read_csv( - total_capacity_csv, index_col=[0, 1, 2] - ) + total_capacity_csv = "results/India/results/TotalCapacityAnnual.csv" + pwr_node_save = "" + trn_node_save = "" + pwr_country_save = "" + trn_country_save = "" + + total_capacity_annual = pd.read_csv(total_capacity_csv, index_col=[0, 1, 2]) - pwr_capacity = calc_pwr_capacity(total_capacity_annual) - trn_capacity = calc_trn_capacity(total_capacity_annual) + pwr_node_capacity = calc_pwr_capacity(total_capacity_annual, country=False) + trn_node_capacity = calc_trn_capacity(total_capacity_annual, country=False) + pwr_country_capacity = calc_pwr_capacity(total_capacity_annual, country=True) + trn_country_capacity = calc_trn_capacity(total_capacity_annual, country=True) - pwr_capacity.to_csv(pwr_save, index=True) - trn_capacity.to_csv(trn_save, index=True) + pwr_node_capacity.to_csv(pwr_node_save, index=True) + trn_node_capacity.to_csv(trn_node_save, index=True) + pwr_country_capacity.to_csv(pwr_country_save, index=True) + trn_country_capacity.to_csv(trn_country_save, index=True) From bd9bf2e235293bb292d6a91b7b10420e718bf74a Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 17 Nov 2024 21:42:58 -0800 Subject: [PATCH 18/27] country transmission correction --- workflow/scripts/osemosys_global/summary/capacity.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/workflow/scripts/osemosys_global/summary/capacity.py b/workflow/scripts/osemosys_global/summary/capacity.py index 796fe824..08479d99 100644 --- a/workflow/scripts/osemosys_global/summary/capacity.py +++ b/workflow/scripts/osemosys_global/summary/capacity.py @@ -14,10 +14,16 @@ def calc_trn_capacity( if country: df["FROM"] = df.index.get_level_values("TECHNOLOGY").str[3:6] df["TO"] = df.index.get_level_values("TECHNOLOGY").str[8:11] + df = df.drop_duplicates(subset=["FROM", "TO"], keep=False) # intercountry else: df["FROM"] = df.index.get_level_values("TECHNOLOGY").str[3:8] df["TO"] = df.index.get_level_values("TECHNOLOGY").str[8:13] + if df.empty: + return pd.DataFrame(columns=["FROM", "TO", "YEAR", "VALUE"]).set_index( + ["FROM", "TO", "YEAR"] + ) + return ( df.reset_index()[["FROM", "TO", "YEAR", "VALUE"]] .groupby(["FROM", "TO", "YEAR"]) From b90ea662fa066a525b08b7ec13e8c9482c2ba5f2 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 17 Nov 2024 22:28:30 -0800 Subject: [PATCH 19/27] costs updated --- workflow/rules/postprocess.smk | 27 ++++--- .../osemosys_global/summary/node_cost.py | 80 ++++++++++++++----- 2 files changed, 76 insertions(+), 31 deletions(-) diff --git a/workflow/rules/postprocess.smk b/workflow/rules/postprocess.smk index b50b1456..a05bd0f9 100644 --- a/workflow/rules/postprocess.smk +++ b/workflow/rules/postprocess.smk @@ -10,10 +10,16 @@ RESULT_FIGURES = [ RESULT_SUMMARIES = [ "TradeFlows", "AnnualEmissionIntensity", - "TransmissionCapacity", - "PowerCapacity", - "GenerationShares", - "NodeCost", + "PowerCapacityNode", + "TransmissionCapacityNode", + "PowerCapacityCountry", + "TransmissionCapacityCountry", + "GenerationSharesNode", + "GenerationSharesCountry", + "PowerCostNode", + "TotalCostNode", + "PowerCostCountry", + "TotalCostCountry", "Metrics" ] @@ -58,7 +64,7 @@ rule visualisation: rule calculate_trade_flows: message: - "Calculating Hourly Trade Flows" + "Calculating Hourly Trade Flows..." params: seasons = config["seasons"], dayparts = config["dayparts"], @@ -85,14 +91,17 @@ rule calculate_carbon_intensity: script: "../scripts/osemosys_global/summary/carbon_intensity.py" -rule calculate_cost_by_node: +rule calculate_costs: message: - "Calculating Cost by Node..." + "Calculating Costs..." input: discounted_cost_by_technology = "results/{scenario}/results/DiscountedCostByTechnology.csv", demand = "results/{scenario}/results/Demand.csv", output: - node_cost = "results/{scenario}/result_summaries/NodeCost.csv", + node_pwr_cost = "results/{scenario}/result_summaries/PowerCostNode.csv", + country_pwr_cost = "results/{scenario}/result_summaries/PowerCostCountry.csv", + node_cost = "results/{scenario}/result_summaries/TotalCostNode.csv", + country_cost = "results/{scenario}/result_summaries/TotalCostCountry.csv", log: log = 'results/{scenario}/logs/node_cost.log' script: @@ -115,7 +124,7 @@ rule calculate_generation_shares: rule calculate_capacity: message: - "Calculating Capacity by Node..." + "Calculating Capacities..." input: total_capacity = "results/{scenario}/results/TotalCapacityAnnual.csv", output: diff --git a/workflow/scripts/osemosys_global/summary/node_cost.py b/workflow/scripts/osemosys_global/summary/node_cost.py index 07e0e2b6..6cccd96c 100644 --- a/workflow/scripts/osemosys_global/summary/node_cost.py +++ b/workflow/scripts/osemosys_global/summary/node_cost.py @@ -4,7 +4,7 @@ from pathlib import Path -def get_tech_cost_per_node(discounted_cost_tech: pd.DataFrame) -> pd.DataFrame: +def get_tech_cost(discounted_cost_tech: pd.DataFrame, country: bool) -> pd.DataFrame: """Only of power generation technologies""" df = discounted_cost_tech.copy() @@ -14,39 +14,57 @@ def get_tech_cost_per_node(discounted_cost_tech: pd.DataFrame) -> pd.DataFrame: & ~(df.index.get_level_values("TECHNOLOGY").str.contains("TRN")) ] - df["NODE"] = df.index.get_level_values("TECHNOLOGY").str[6:11] + if country: + r = "COUNTRY" + df[r] = df.index.get_level_values("TECHNOLOGY").str[6:9] + else: + r = "NODE" + df[r] = df.index.get_level_values("TECHNOLOGY").str[6:11] return ( - df.reset_index()[["REGION", "NODE", "YEAR", "VALUE"]] - .groupby(["REGION", "NODE", "YEAR"]) + df.reset_index()[["REGION", r, "YEAR", "VALUE"]] + .groupby(["REGION", r, "YEAR"]) .sum() ) -def get_storage_cost_per_node(discounted_cost_storage: pd.DataFrame) -> pd.DataFrame: +def get_storage_cost(discounted_cost_storage: pd.DataFrame, country: bool) -> pd.DataFrame: df = discounted_cost_storage.copy() - df["NODE"] = df.index.get_level_values("STORAGE").str[3:8] + + if country: + r = "COUNTRY" + df[r] = df.index.get_level_values("STORAGE").str[3:6] + else: + r = "NODE" + df[r] = df.index.get_level_values("STORAGE").str[3:8] + return ( - df.reset_index()[["REGION", "NODE", "YEAR", "VALUE"]] - .groupby(["REGION", "NODE", "YEAR"]) + df.reset_index()[["REGION", r, "YEAR", "VALUE"]] + .groupby(["REGION", r, "YEAR"]) .sum() ) -def get_demand_per_node(demand: pd.DataFrame) -> pd.DataFrame: +def get_demand(demand: pd.DataFrame, country: bool) -> pd.DataFrame: df = demand.copy() df = df[df.index.get_level_values("FUEL").str.startswith("ELC")] - df["NODE"] = df.index.get_level_values("FUEL").str[3:8] + + if country: + r = "COUNTRY" + df[r] = df.index.get_level_values("FUEL").str[3:6] + else: + r = "NODE" + df[r] = df.index.get_level_values("FUEL").str[3:8] return ( - df.reset_index()[["REGION", "NODE", "YEAR", "VALUE"]] - .groupby(["REGION", "NODE", "YEAR"]) + df.reset_index()[["REGION", r, "YEAR", "VALUE"]] + .groupby(["REGION", r, "YEAR"]) .sum() ) -def get_pwr_cost_per_node(demand: pd.DataFrame, cost: pd.DataFrame) -> pd.DataFrame: +def get_pwr_cost(demand: pd.DataFrame, cost: pd.DataFrame) -> pd.DataFrame: """Gets power generation cost per node in $/MWh""" # ($M / PJ) (1PJ / 1000 TJ) (1TJ / 1000 GJ) (1GJ / 1000 MJ) ($1000000 / $M) (3600sec / hr) @@ -60,13 +78,19 @@ def get_pwr_cost_per_node(demand: pd.DataFrame, cost: pd.DataFrame) -> pd.DataFr snakemake.input.discounted_cost_by_technology ) demand_csv = snakemake.input.demand - save = snakemake.output.node_cost + power_cost_node_csv = snakemake.output.node_pwr_cost + power_cost_country_csv = snakemake.output.country_pwr_cost + total_cost_node_csv = snakemake.output.node_cost + total_cost_country_csv = snakemake.output.country_cost else: discounted_cost_by_technology_csv = ( "results/India/results/DiscountedCostByTechnology.csv" ) demand_csv = "results/India/results/Demand.csv" - save = "results/India/results/NodeCost.csv" + power_cost_node_csv = "results/India/result_summaries/NodePowerCost.csv" + power_cost_country_csv = "results/India/result_summaries/CountryPowerCost.csv" + total_cost_node_csv = "results/India/result_summaries/NodeCost.csv" + total_cost_country_csv = "results/India/result_summaries/CountryCost.csv" discounted_cost_by_technology = pd.read_csv( discounted_cost_by_technology_csv, index_col=[0, 1, 2] @@ -87,13 +111,25 @@ def get_pwr_cost_per_node(demand: pd.DataFrame, cost: pd.DataFrame) -> pd.DataFr columns=["REGION", "STORAGE", "YEAR", "VALUE"] ).set_index(["REGION", "STORAGE", "YEAR"]) - tech_cost = get_tech_cost_per_node(discounted_cost_by_technology) - storage_cost = get_storage_cost_per_node(discounted_cost_by_storage) - + + # node level metrics + + tech_cost = get_tech_cost(discounted_cost_by_technology, country=False) + storage_cost = get_storage_cost(discounted_cost_by_storage, country=False) cost = tech_cost.add(storage_cost, fill_value=0) - - demand = get_demand_per_node(demand_raw) + demand = get_demand(demand_raw, country=False) + pwr_cost = get_pwr_cost(demand, cost) - df = get_pwr_cost_per_node(demand, cost) + pwr_cost.to_csv(power_cost_node_csv, index=True) + cost.to_csv(total_cost_node_csv, index=True) + + # country level metrics + + tech_cost = get_tech_cost(discounted_cost_by_technology, country=True) + storage_cost = get_storage_cost(discounted_cost_by_storage, country=True) + cost = tech_cost.add(storage_cost, fill_value=0) + demand = get_demand(demand_raw, country=True) + pwr_cost = get_pwr_cost(demand, cost) - df.to_csv(save, index=True) + pwr_cost.to_csv(power_cost_country_csv, index=True) + cost.to_csv(total_cost_country_csv, index=True) From fc2a0d17b022746caea98e12f0b91d0bf1cfb5d8 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 17 Nov 2024 22:28:44 -0800 Subject: [PATCH 20/27] fix yaml config --- resources/otoole.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/otoole.yaml b/resources/otoole.yaml index dd44716e..5a2f5c41 100644 --- a/resources/otoole.yaml +++ b/resources/otoole.yaml @@ -374,7 +374,7 @@ DiscountedCostByTechnology: type: result dtype: float default: 0 -DiscountedOperatingCost: +DiscountedOperationalCost: indices: [REGION, TECHNOLOGY, YEAR] type: result dtype: float From f73bddf58c20d84f1720d3d576a18cd1f1e8692f Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 17 Nov 2024 22:29:53 -0800 Subject: [PATCH 21/27] config update --- workflow/snakefile | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/snakefile b/workflow/snakefile index 62f00124..5c6b220b 100644 --- a/workflow/snakefile +++ b/workflow/snakefile @@ -32,7 +32,6 @@ def get_otoole_data(otoole_config: str, var: str) -> list[str]: "StorageLevelYearStart", "StorageLevelYearFinish", "Trade", - "DiscountedOperatingCost", "TotalDiscountedStorageCost", "ModelPeriodEmissions", "StorageLowerLimit", From 03249193e988c49156c67420fb20a3c4b74e12c6 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Mon, 18 Nov 2024 11:22:26 -0800 Subject: [PATCH 22/27] emission intensity updated to use production --- workflow/rules/postprocess.smk | 4 +- .../summary/carbon_intensity.py | 61 +++++++++++++------ 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/workflow/rules/postprocess.smk b/workflow/rules/postprocess.smk index a05bd0f9..f3e66054 100644 --- a/workflow/rules/postprocess.smk +++ b/workflow/rules/postprocess.smk @@ -81,8 +81,10 @@ rule calculate_trade_flows: rule calculate_carbon_intensity: message: "Calculating Carbon Intensity..." + params: + storage = config['storage_parameters'], input: - demand = "results/{scenario}/results/Demand.csv", + production_by_technology = "results/{scenario}/results/ProductionByTechnology.csv", annual_emissions = "results/{scenario}/results/AnnualEmissions.csv", output: emission_intensity = "results/{scenario}/result_summaries/AnnualEmissionIntensity.csv", diff --git a/workflow/scripts/osemosys_global/summary/carbon_intensity.py b/workflow/scripts/osemosys_global/summary/carbon_intensity.py index af95c629..e2ec7648 100644 --- a/workflow/scripts/osemosys_global/summary/carbon_intensity.py +++ b/workflow/scripts/osemosys_global/summary/carbon_intensity.py @@ -5,63 +5,84 @@ """ import pandas as pd +from typing import Optional -def format_demand(demand: pd.DataFrame) -> pd.DataFrame: +def format_production( + production: pd.DataFrame, + exclude: Optional[list[str]] = None, +) -> pd.DataFrame: + + df = production.copy() + + df = df[ + (df.index.get_level_values("TECHNOLOGY").str.startswith("PWR")) + & ~(df.index.get_level_values("TECHNOLOGY").str.contains("TRN")) + ].copy() - df = demand.copy() + df["TECH"] = df.index.get_level_values("TECHNOLOGY").str[3:6] + df["COUNTRY"] = df.index.get_level_values("TECHNOLOGY").str[6:9] - df["CTRY"] = df.FUEL.str[3:6] - df = df.drop(columns=["TIMESLICE", "FUEL"]) - return df.groupby(["REGION", "CTRY", "YEAR"]).sum() + if exclude: + df = df[~df.TECH.isin(exclude)].copy() + + return ( + df.reset_index()[["REGION", "COUNTRY", "YEAR", "VALUE"]] + .groupby(["REGION", "COUNTRY", "YEAR"]) + .sum() + ) def format_emissions(annual_emissions: pd.DataFrame) -> pd.DataFrame: - df = annual_emissions.copy() + df = annual_emissions.copy().reset_index() - df["CTRY"] = df.EMISSION.str[3:6] - return df.groupby(["REGION", "EMISSION", "CTRY", "YEAR"]).sum() + df["COUNTRY"] = df.EMISSION.str[3:6] + return df.groupby(["REGION", "EMISSION", "COUNTRY", "YEAR"]).sum() def calculate_emission_intensity( - demand: pd.DataFrame, emissions: pd.DataFrame + production: pd.DataFrame, emissions: pd.DataFrame ) -> pd.DataFrame: - d = demand.copy().rename(columns={"VALUE": "DEMAND_PJ"}) + p = production.copy().rename(columns={"VALUE": "PROD_PJ"}) e = emissions.copy().rename(columns={"VALUE": "EMISSIONS_MT"}) - df = d.join(e).fillna(0) - df = df.droplevel("CTRY") # emission retains country + df = p.join(e).fillna(0) + df = df.droplevel("COUNTRY") # emission retains country # 1 PJ (1000 TJ / PJ) (1000 GJ / TJ) (1000 MJ / GJ) (1000 kJ / MJ) (hr / 3600sec) - df["DEMAND_kwh"] = df.DEMAND_PJ.mul(1000).mul(1000).mul(1000).mul(1000).div(3600) + df["PROD_kwh"] = df.PROD_PJ.mul(1000).mul(1000).mul(1000).mul(1000).div(3600) # 1 MT (1,000,000 T / MT) (1000 kg / T) (1000g / kg) df["EMISSIONS_g"] = df.EMISSIONS_MT.mul(1000000).mul(1000).mul(1000) # intensity in g/kwh - df["VALUE"] = df.EMISSIONS_g.div(df.DEMAND_kwh) + df["VALUE"] = df.EMISSIONS_g.div(df.PROD_kwh) return df["VALUE"].to_frame() if __name__ == "__main__": if "snakemake" in globals(): - demand_csv = snakemake.input.demand + production_csv = snakemake.input.production_by_technology annual_emissions_csv = snakemake.input.annual_emissions save = snakemake.output.emission_intensity + storage = snakemake.params.storage else: - demand_csv = "results/India/results/Demand.csv" + production_csv = "results/India/results/ProductionByTechnology.csv" annual_emissions_csv = "results/India/results/AnnualEmissions.csv" save = "results/India/results/AnnualEmissionIntensity.csv" + storage = {"SDS": [], "LDS": []} + + exclusions = list(storage) - demand = pd.read_csv(demand_csv) - annual_emissions = pd.read_csv(annual_emissions_csv) + production = pd.read_csv(production_csv, index_col=[0, 1, 2, 3, 4]) + annual_emissions = pd.read_csv(annual_emissions_csv, index_col=[0, 1, 2]) - demand = format_demand(demand) + production = format_production(production, exclusions) emissions = format_emissions(annual_emissions) - emission_intensity = calculate_emission_intensity(demand, emissions).round(2) + emission_intensity = calculate_emission_intensity(production, emissions).round(2) emission_intensity.to_csv(save, index=True) From 41cf1a813e956d389465e4af52b942ac9a735fe8 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Mon, 18 Nov 2024 13:01:19 -0800 Subject: [PATCH 23/27] correct emission result --- resources/otoole.yaml | 6 ------ workflow/snakefile | 18 ++++++++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/resources/otoole.yaml b/resources/otoole.yaml index 5a2f5c41..7339e7b2 100644 --- a/resources/otoole.yaml +++ b/resources/otoole.yaml @@ -322,12 +322,6 @@ AnnualTechnologyEmission: type: result dtype: float default: 0 -AnnualTechnologyEmissionsPenalty: - indices: [REGION, TECHNOLOGY, YEAR] - type: result - dtype: float - default: 0 - short_name: AnnualTechEmissionsPenalty AnnualTechnologyEmissionByMode: indices: [REGION, TECHNOLOGY, EMISSION, MODE_OF_OPERATION, YEAR] type: result diff --git a/workflow/snakefile b/workflow/snakefile index 5c6b220b..bdc0c2fc 100644 --- a/workflow/snakefile +++ b/workflow/snakefile @@ -20,21 +20,27 @@ def get_otoole_data(otoole_config: str, var: str) -> list[str]: results = [x for x in otoole if otoole[x]["type"] == var] # no result calcualtions - # or if no result present (ie. no storage runs) missing = [ + # storage investment not garunteed "NewStorageCapacity", - "NumberOfNewTechnologyUnits", + "TotalDiscountedStorageCost", "SalvageValueStorage", - "StorageLevelDayTypeStart", - "StorageLevelDayTypeFinish", + "CapitalInvestmentStorage" + "DiscountedCapitalInvestmentStorage", + "DiscountedSalvageValueStorage" + # storage levels not garunteed "StorageLevelSeasonStart", "StorageLevelSeasonFinish", "StorageLevelYearStart", "StorageLevelYearFinish", + "StorageLevelDayTypeStart", + "StorageLevelDayTypeFinish", + "StorageLowerLimit", + "StorageUpperLimit", + # other missing calcs + "NumberOfNewTechnologyUnits", "Trade", - "TotalDiscountedStorageCost", "ModelPeriodEmissions", - "StorageLowerLimit", ] return [x for x in results if x not in missing] From 56f1d3d64e91f3a579829ef9cbf1682b28d544cc Mon Sep 17 00:00:00 2001 From: maartenbrinkerink <65602545+maartenbrinkerink@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:18:43 -0500 Subject: [PATCH 24/27] Comma fixes --- workflow/snakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/snakefile b/workflow/snakefile index bdc0c2fc..817d0ef8 100644 --- a/workflow/snakefile +++ b/workflow/snakefile @@ -25,9 +25,9 @@ def get_otoole_data(otoole_config: str, var: str) -> list[str]: "NewStorageCapacity", "TotalDiscountedStorageCost", "SalvageValueStorage", - "CapitalInvestmentStorage" + "CapitalInvestmentStorage", "DiscountedCapitalInvestmentStorage", - "DiscountedSalvageValueStorage" + "DiscountedSalvageValueStorage", # storage levels not garunteed "StorageLevelSeasonStart", "StorageLevelSeasonFinish", From 277bad26ab3a83d6a31ed7eb0fdce9f32ddebbf2 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Mon, 18 Nov 2024 16:32:39 -0800 Subject: [PATCH 25/27] correct exception handeling --- workflow/rules/postprocess.smk | 2 +- .../summary/{node_cost.py => costs.py} | 23 +++++++++++-------- workflow/snakefile | 5 ++-- 3 files changed, 17 insertions(+), 13 deletions(-) rename workflow/scripts/osemosys_global/summary/{node_cost.py => costs.py} (94%) diff --git a/workflow/rules/postprocess.smk b/workflow/rules/postprocess.smk index f3e66054..4fe0a506 100644 --- a/workflow/rules/postprocess.smk +++ b/workflow/rules/postprocess.smk @@ -107,7 +107,7 @@ rule calculate_costs: log: log = 'results/{scenario}/logs/node_cost.log' script: - "../scripts/osemosys_global/summary/node_cost.py" + "../scripts/osemosys_global/summary/costs.py" rule calculate_generation_shares: message: diff --git a/workflow/scripts/osemosys_global/summary/node_cost.py b/workflow/scripts/osemosys_global/summary/costs.py similarity index 94% rename from workflow/scripts/osemosys_global/summary/node_cost.py rename to workflow/scripts/osemosys_global/summary/costs.py index 6cccd96c..1273a5c4 100644 --- a/workflow/scripts/osemosys_global/summary/node_cost.py +++ b/workflow/scripts/osemosys_global/summary/costs.py @@ -27,29 +27,33 @@ def get_tech_cost(discounted_cost_tech: pd.DataFrame, country: bool) -> pd.DataF .sum() ) -def get_storage_cost(discounted_cost_storage: pd.DataFrame, country: bool) -> pd.DataFrame: + +def get_storage_cost( + discounted_cost_storage: pd.DataFrame, country: bool +) -> pd.DataFrame: df = discounted_cost_storage.copy() - + if country: r = "COUNTRY" df[r] = df.index.get_level_values("STORAGE").str[3:6] else: r = "NODE" df[r] = df.index.get_level_values("STORAGE").str[3:8] - + return ( df.reset_index()[["REGION", r, "YEAR", "VALUE"]] .groupby(["REGION", r, "YEAR"]) .sum() ) + def get_demand(demand: pd.DataFrame, country: bool) -> pd.DataFrame: df = demand.copy() df = df[df.index.get_level_values("FUEL").str.startswith("ELC")] - + if country: r = "COUNTRY" df[r] = df.index.get_level_values("FUEL").str[3:6] @@ -106,13 +110,12 @@ def get_pwr_cost(demand: pd.DataFrame, cost: pd.DataFrame) -> pd.DataFrame: discounted_cost_by_storage = pd.read_csv( discounted_cost_by_storage_csv, index_col=[0, 1, 2] ) - except FileExistsError: - discounted_cost_by_storage_raw = pd.DataFrame( + except FileNotFoundError: + discounted_cost_by_storage = pd.DataFrame( columns=["REGION", "STORAGE", "YEAR", "VALUE"] ).set_index(["REGION", "STORAGE", "YEAR"]) - - # node level metrics + # node level metrics tech_cost = get_tech_cost(discounted_cost_by_technology, country=False) storage_cost = get_storage_cost(discounted_cost_by_storage, country=False) @@ -123,8 +126,8 @@ def get_pwr_cost(demand: pd.DataFrame, cost: pd.DataFrame) -> pd.DataFrame: pwr_cost.to_csv(power_cost_node_csv, index=True) cost.to_csv(total_cost_node_csv, index=True) - # country level metrics - + # country level metrics + tech_cost = get_tech_cost(discounted_cost_by_technology, country=True) storage_cost = get_storage_cost(discounted_cost_by_storage, country=True) cost = tech_cost.add(storage_cost, fill_value=0) diff --git a/workflow/snakefile b/workflow/snakefile index 817d0ef8..b113bab2 100644 --- a/workflow/snakefile +++ b/workflow/snakefile @@ -21,14 +21,15 @@ def get_otoole_data(otoole_config: str, var: str) -> list[str]: # no result calcualtions missing = [ - # storage investment not garunteed + # storage investment not guaranteed "NewStorageCapacity", "TotalDiscountedStorageCost", "SalvageValueStorage", "CapitalInvestmentStorage", "DiscountedCapitalInvestmentStorage", "DiscountedSalvageValueStorage", - # storage levels not garunteed + "DiscountedCostByStorage", + # storage levels not guaranteed "StorageLevelSeasonStart", "StorageLevelSeasonFinish", "StorageLevelYearStart", From f5e3101fffb83fddba2039f50837b3fe2c4ee8b9 Mon Sep 17 00:00:00 2001 From: maartenbrinkerink <65602545+maartenbrinkerink@users.noreply.github.com> Date: Tue, 19 Nov 2024 09:49:03 -0500 Subject: [PATCH 26/27] Country level transmission capacity fix --- workflow/scripts/osemosys_global/summary/capacity.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/workflow/scripts/osemosys_global/summary/capacity.py b/workflow/scripts/osemosys_global/summary/capacity.py index 08479d99..d46a4960 100644 --- a/workflow/scripts/osemosys_global/summary/capacity.py +++ b/workflow/scripts/osemosys_global/summary/capacity.py @@ -14,7 +14,7 @@ def calc_trn_capacity( if country: df["FROM"] = df.index.get_level_values("TECHNOLOGY").str[3:6] df["TO"] = df.index.get_level_values("TECHNOLOGY").str[8:11] - df = df.drop_duplicates(subset=["FROM", "TO"], keep=False) # intercountry + df = df.loc[df['FROM'] != df['TO']] # intercountry else: df["FROM"] = df.index.get_level_values("TECHNOLOGY").str[3:8] df["TO"] = df.index.get_level_values("TECHNOLOGY").str[8:13] @@ -68,10 +68,10 @@ def calc_pwr_capacity( trn_country_save = snakemake.output.transmission_capacity_country else: total_capacity_csv = "results/India/results/TotalCapacityAnnual.csv" - pwr_node_save = "" - trn_node_save = "" - pwr_country_save = "" - trn_country_save = "" + pwr_node_save = "results/India/result_summaries/PowerCapacityNode.csv" + trn_node_save = "results/India/result_summaries/TransmissionCapacityNode.csv" + pwr_country_save = "results/India/result_summaries/PowerCapacityCountry.csv" + trn_country_save = "results/India/result_summaries/TransmissionCapacityCountry.csv" total_capacity_annual = pd.read_csv(total_capacity_csv, index_col=[0, 1, 2]) From 8f7af79d19f02e4aa8d4b9e477d88a34d4bac765 Mon Sep 17 00:00:00 2001 From: maartenbrinkerink <65602545+maartenbrinkerink@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:33:47 -0500 Subject: [PATCH 27/27] Added country level flows and diversified imports/exports/net trade/total trade --- workflow/rules/postprocess.smk | 24 ++++- .../osemosys_global/summary/trade_flows.py | 102 ++++++++++++++++-- 2 files changed, 116 insertions(+), 10 deletions(-) diff --git a/workflow/rules/postprocess.smk b/workflow/rules/postprocess.smk index 4fe0a506..6f105627 100644 --- a/workflow/rules/postprocess.smk +++ b/workflow/rules/postprocess.smk @@ -8,7 +8,16 @@ RESULT_FIGURES = [ ] RESULT_SUMMARIES = [ - "TradeFlows", + "TradeFlowsNode", + "TradeFlowsCountry", + "AnnualNetTradeFlowsNode", + "AnnualNetTradeFlowsCountry", + "AnnualImportTradeFlowsNode", + "AnnualImportTradeFlowsCountry", + "AnnualExportTradeFlowsNode", + "AnnualExportTradeFlowsCountry", + "AnnualTotalTradeFlowsNode", + "AnnualTotalTradeFlowsCountry", "AnnualEmissionIntensity", "PowerCapacityNode", "TransmissionCapacityNode", @@ -64,7 +73,7 @@ rule visualisation: rule calculate_trade_flows: message: - "Calculating Hourly Trade Flows..." + "Calculating Trade Flows..." params: seasons = config["seasons"], dayparts = config["dayparts"], @@ -72,7 +81,16 @@ rule calculate_trade_flows: input: activity_by_mode = "results/{scenario}/results/TotalAnnualTechnologyActivityByMode.csv", output: - trade_flows = "results/{scenario}/result_summaries/TradeFlows.csv", + node_trade_flows = "results/{scenario}/result_summaries/TradeFlowsNode.csv", + country_trade_flows = "results/{scenario}/result_summaries/TradeFlowsCountry.csv", + annual_net_node_trade_flows = "results/{scenario}/result_summaries/AnnualNetTradeFlowsNode.csv", + annual_net_country_trade_flows = "results/{scenario}/result_summaries/AnnualNetTradeFlowsCountry.csv", + annual_import_node_trade_flows = "results/{scenario}/result_summaries/AnnualImportTradeFlowsNode.csv", + annual_import_country_trade_flows = "results/{scenario}/result_summaries/AnnualImportTradeFlowsCountry.csv", + annual_export_node_trade_flows = "results/{scenario}/result_summaries/AnnualExportTradeFlowsNode.csv", + annual_export_country_trade_flows = "results/{scenario}/result_summaries/AnnualExportTradeFlowsCountry.csv", + annual_total_node_trade_flows = "results/{scenario}/result_summaries/AnnualTotalTradeFlowsNode.csv", + annual_total_country_trade_flows = "results/{scenario}/result_summaries/AnnualTotalTradeFlowsCountry.csv", log: log = "results/{scenario}/logs/trade_flows.log" script: diff --git a/workflow/scripts/osemosys_global/summary/trade_flows.py b/workflow/scripts/osemosys_global/summary/trade_flows.py index 5fe3116d..a9229931 100644 --- a/workflow/scripts/osemosys_global/summary/trade_flows.py +++ b/workflow/scripts/osemosys_global/summary/trade_flows.py @@ -1,4 +1,4 @@ -"""Calcualtes Transmission Capacity per node""" +"""Calcualtes Transmission Flows""" import pandas as pd import itertools @@ -22,7 +22,7 @@ def apply_timeshift(x: int, timeshift: int) -> int: return x -def get_trade_flows( +def get_trade_flows_node( activity_by_mode: pd.DataFrame, seasons_raw: dict[str, list[int]], dayparts_raw: dict[str, list[int]], @@ -166,11 +166,72 @@ def get_trade_flows( ) return df +def get_trade_flows_country( + trade_flows_node: pd.DataFrame, +) -> pd.DataFrame: + + df = trade_flows_node.copy() + + df["NODE_1"] = df['NODE_1'].str[0:3] + 'XX' + df["NODE_2"] = df['NODE_2'].str[0:3] + 'XX' + df = df.loc[df["NODE_1"] != df["NODE_2"]] # intercountry + + return(df.groupby(["YEAR", "MONTH", "HOUR", "NODE_1", "NODE_2"], as_index = False, + observed = False).sum()) + +def get_net_annual_flows( + trade_flows_ts: pd.DataFrame, +) -> pd.DataFrame: + + df = trade_flows_ts.copy() + + return(df.groupby(["YEAR", "NODE_1", "NODE_2"], as_index = False, + observed = False)['VALUE'].sum()) + +def get_import_annual_flows( + trade_flows_ts: pd.DataFrame, +) -> pd.DataFrame: + + df = trade_flows_ts.copy() + df = df.loc[df['VALUE'] < 0] + df['VALUE'] = df['VALUE'] * -1 + + return(df.groupby(["YEAR", "NODE_1", "NODE_2"], as_index = False, + observed = False)['VALUE'].sum()) + +def get_export_annual_flows( + trade_flows_ts: pd.DataFrame, +) -> pd.DataFrame: + + df = trade_flows_ts.copy() + df = df.loc[df['VALUE'] > 0] + + return(df.groupby(["YEAR", "NODE_1", "NODE_2"], as_index = False, + observed = False)['VALUE'].sum()) + +def get_total_annual_flows( + trade_flows_ts: pd.DataFrame, +) -> pd.DataFrame: + + df = trade_flows_ts.copy() + df['VALUE'] = df['VALUE'].abs() + + return(df.groupby(["YEAR", "NODE_1", "NODE_2"], as_index = False, + observed = False)['VALUE'].sum()) if __name__ == "__main__": if "snakemake" in globals(): activity_by_mode_csv = snakemake.input.activity_by_mode - save = snakemake.output.trade_flows + trade_flows_node_save = snakemake.output.node_trade_flows + trade_flows_country_save = snakemake.output.country_trade_flows + annual_net_trade_flows_node_save = snakemake.output.annual_net_node_trade_flows + annual_net_trade_flows_country_save = snakemake.output.annual_net_country_trade_flows + annual_import_trade_flows_node_save = snakemake.output.annual_import_node_trade_flows + annual_import_trade_flows_country_save = snakemake.output.annual_import_country_trade_flows + annual_export_trade_flows_node_save = snakemake.output.annual_export_node_trade_flows + annual_export_trade_flows_country_save = snakemake.output.annual_export_country_trade_flows + annual_total_trade_flows_node_save = snakemake.output.annual_total_node_trade_flows + annual_total_trade_flows_country_save = snakemake.output.annual_total_country_trade_flows seasons = snakemake.params.seasons dayparts = snakemake.params.dayparts timeshift = snakemake.params.timeshift @@ -178,13 +239,40 @@ def get_trade_flows( activity_by_mode_csv = ( "results/India/results/TotalAnnualTechnologyActivityByMode.csv" ) - save = "results/India/results/TradeFlows.csv" + trade_flows_node_save = "results/India/result_summaries/TradeFlowsNode.csv" + trade_flows_country_save = "results/India/result_summaries/TradeFlowsCountry.csv" + annual_net_trade_flows_node_save = "results/India/result_summaries/AnnualNetTradeFlowsNode.csv" + annual_net_trade_flows_country_save = "results/India/result_summaries/AnnualNetTradeFlowsCountry.csv" + annual_import_trade_flows_node_save = "results/India/result_summaries/AnnualImportTradeFlowsNode.csv" + annual_import_trade_flows_country_save = "results/India/result_summaries/AnnualImportTradeFlowsCountry.csv" + annual_export_trade_flows_node_save = "results/India/result_summaries/AnnualExportTradeFlowsNode.csv" + annual_export_trade_flows_country_save = "results/India/result_summaries/AnnualExportTradeFlowsCountry.csv" + annual_total_trade_flows_node_save = "results/India/result_summaries/AnnualTotalTradeFlowsNode.csv" + annual_total_trade_flows_country_save = "results/India/result_summaries/AnnualTotalTradeFlowsCountry.csv" seasons = {"S1": [1, 2, 3, 4, 5, 6], "S2": [7, 8, 9, 10, 11, 12]} dayparts = {"D1": [1, 7], "D2": [7, 13], "D3": [13, 19], "D4": [19, 25]} timeshift = 0 activity_by_mode = pd.read_csv(activity_by_mode_csv, index_col=[0, 1, 2, 3, 4]) - df = get_trade_flows(activity_by_mode, seasons, dayparts, timeshift) - - df.to_csv(save, index=False) + trade_flows_node = get_trade_flows_node(activity_by_mode, seasons, dayparts, timeshift) + trade_flows_country = get_trade_flows_country(trade_flows_node) + annual_net_trade_flows_node = get_net_annual_flows(trade_flows_node) + annual_net_trade_flows_country = get_net_annual_flows(trade_flows_country) + annual_import_trade_flows_node = get_import_annual_flows(trade_flows_node) + annual_import_trade_flows_country = get_import_annual_flows(trade_flows_country) + annual_export_trade_flows_node = get_export_annual_flows(trade_flows_node) + annual_export_trade_flows_country = get_export_annual_flows(trade_flows_country) + annual_total_trade_flows_node = get_total_annual_flows(trade_flows_node) + annual_total_trade_flows_country = get_total_annual_flows(trade_flows_country) + + trade_flows_node.to_csv(trade_flows_node_save, index=False) + trade_flows_country.to_csv(trade_flows_country_save, index=False) + annual_net_trade_flows_node.to_csv(annual_net_trade_flows_node_save, index=False) + annual_net_trade_flows_country.to_csv(annual_net_trade_flows_country_save, index=False) + annual_import_trade_flows_node.to_csv(annual_import_trade_flows_node_save, index=False) + annual_import_trade_flows_country.to_csv(annual_import_trade_flows_country_save, index=False) + annual_export_trade_flows_node.to_csv(annual_export_trade_flows_node_save, index=False) + annual_export_trade_flows_country.to_csv(annual_export_trade_flows_country_save, index=False) + annual_total_trade_flows_node.to_csv(annual_total_trade_flows_node_save, index=False) + annual_total_trade_flows_country.to_csv(annual_total_trade_flows_country_save, index=False)