From d3a7a6bc57a8e1144744360b8a9dd8312df7b159 Mon Sep 17 00:00:00 2001 From: Ana Mileva Date: Fri, 22 Nov 2024 17:55:20 -0800 Subject: [PATCH] Market prices by market and iteration type (#1181) --- db/csvs_test_examples/csv_structure.csv | 2 +- .../markets/prices/1_base_prices.csv | 5 +- .../price_profiles/1_base_price_profiles.csv | 19 ---- .../Market_Hub-1-base_price_profiles.csv | 7 ++ .../Market_Hub2-1-base_price_profiles.csv | 13 +++ db/db_schema.sql | 17 ++-- gridpath/system/markets/prices.py | 96 ++++++++++++------- 7 files changed, 93 insertions(+), 66 deletions(-) delete mode 100644 db/csvs_test_examples/markets/prices/price_profiles/1_base_price_profiles.csv create mode 100644 db/csvs_test_examples/markets/prices/price_profiles/Market_Hub-1-base_price_profiles.csv create mode 100644 db/csvs_test_examples/markets/prices/price_profiles/Market_Hub2-1-base_price_profiles.csv diff --git a/db/csvs_test_examples/csv_structure.csv b/db/csvs_test_examples/csv_structure.csv index f959ec495..ef49e9dac 100644 --- a/db/csvs_test_examples/csv_structure.csv +++ b/db/csvs_test_examples/csv_structure.csv @@ -155,7 +155,7 @@ reliability/local_capacity/system_local_capacity_requirement,local_capacity,loca markets/geography_markets,markets,market_scenario_id,geography_markets,simple,,0,,,,, markets/load_zone_markets,markets,load_zone_market_scenario_id,load_zone_markets,simple,,0,,,,, markets/prices,markets,market_price_scenario_id,market_prices,simple,,0,,,,, -markets/prices/price_profiles,markets,market_price_profile_scenario_id,market_price_profiles,simple,,0,,,,, +markets/prices/price_profiles,markets,market_price_profile_scenario_id,market_price_profiles,simple,,1,market,,,market_price_scenario_id,market_prices markets/volume,markets,market_volume_scenario_id,market_volume,simple,,0,,,,, water/water_system_params,water,water_system_params_scenario_id,system_water_system_params,simple,,0,,,,, water/water_network,water,water_network_scenario_id,geography_water_network,simple,,0,,,,, diff --git a/db/csvs_test_examples/markets/prices/1_base_prices.csv b/db/csvs_test_examples/markets/prices/1_base_prices.csv index 1d14178ac..7ed61a0bf 100644 --- a/db/csvs_test_examples/markets/prices/1_base_prices.csv +++ b/db/csvs_test_examples/markets/prices/1_base_prices.csv @@ -1,2 +1,3 @@ -market_price_profile_scenario_id,varies_by_weather_iteration,varies_by_hydro_iteration -1,0,0 +market,market_price_profile_scenario_id,varies_by_weather_iteration,varies_by_hydro_iteration +Market_Hub,1,0,0 +Market_Hub2,1,0,0 \ No newline at end of file diff --git a/db/csvs_test_examples/markets/prices/price_profiles/1_base_price_profiles.csv b/db/csvs_test_examples/markets/prices/price_profiles/1_base_price_profiles.csv deleted file mode 100644 index b28692212..000000000 --- a/db/csvs_test_examples/markets/prices/price_profiles/1_base_price_profiles.csv +++ /dev/null @@ -1,19 +0,0 @@ -weather_iteration,hydro_iteration,stage_id,timepoint,market,market_price -0,0,1,20200101,Market_Hub,10 -0,0,1,20200102,Market_Hub,-10 -0,0,1,20200201,Market_Hub,10 -0,0,1,20200202,Market_Hub,-10 -0,0,1,20200301,Market_Hub,10 -0,0,1,20200302,Market_Hub,-10 -0,0,1,20200101,Market_Hub2,1 -0,0,1,20200102,Market_Hub2,2 -0,0,1,20200201,Market_Hub2,3 -0,0,1,20200202,Market_Hub2,2 -0,0,1,20200301,Market_Hub2,1 -0,0,1,20200302,Market_Hub2,2 -0,0,2,20200101,Market_Hub2,3 -0,0,2,20200102,Market_Hub2,2 -0,0,2,20200201,Market_Hub2,1 -0,0,2,20200202,Market_Hub2,2 -0,0,2,20200301,Market_Hub2,3 -0,0,2,20200302,Market_Hub2,2 diff --git a/db/csvs_test_examples/markets/prices/price_profiles/Market_Hub-1-base_price_profiles.csv b/db/csvs_test_examples/markets/prices/price_profiles/Market_Hub-1-base_price_profiles.csv new file mode 100644 index 000000000..eb99359c1 --- /dev/null +++ b/db/csvs_test_examples/markets/prices/price_profiles/Market_Hub-1-base_price_profiles.csv @@ -0,0 +1,7 @@ +weather_iteration,hydro_iteration,stage_id,timepoint,market_price +0,0,1,20200101,10 +0,0,1,20200102,-10 +0,0,1,20200201,10 +0,0,1,20200202,-10 +0,0,1,20200301,10 +0,0,1,20200302,-10 \ No newline at end of file diff --git a/db/csvs_test_examples/markets/prices/price_profiles/Market_Hub2-1-base_price_profiles.csv b/db/csvs_test_examples/markets/prices/price_profiles/Market_Hub2-1-base_price_profiles.csv new file mode 100644 index 000000000..9fe91d0af --- /dev/null +++ b/db/csvs_test_examples/markets/prices/price_profiles/Market_Hub2-1-base_price_profiles.csv @@ -0,0 +1,13 @@ +weather_iteration,hydro_iteration,stage_id,timepoint,market_price +0,0,1,20200101,1 +0,0,1,20200102,2 +0,0,1,20200201,3 +0,0,1,20200202,2 +0,0,1,20200301,1 +0,0,1,20200302,2 +0,0,2,20200101,3 +0,0,2,20200102,2 +0,0,2,20200201,1 +0,0,2,20200202,2 +0,0,2,20200301,3 +0,0,2,20200302,2 \ No newline at end of file diff --git a/db/db_schema.sql b/db/db_schema.sql index 76f78ba6a..9605c03c2 100644 --- a/db/db_schema.sql +++ b/db/db_schema.sql @@ -933,10 +933,12 @@ DROP TABLE IF EXISTS inputs_market_prices; CREATE TABLE inputs_market_prices ( market_price_scenario_id INTEGER, + market TEXT, market_price_profile_scenario_id INTEGER, varies_by_weather_iteration INTEGER, varies_by_hydro_iteration INTEGER, - PRIMARY KEY (market_price_scenario_id, market_price_profile_scenario_id), + PRIMARY KEY (market_price_scenario_id, market, + market_price_profile_scenario_id), FOREIGN KEY (market_price_scenario_id) REFERENCES subscenarios_market_prices (market_price_scenario_id) ); @@ -944,24 +946,25 @@ CREATE TABLE inputs_market_prices DROP TABLE IF EXISTS subscenarios_market_price_profiles; CREATE TABLE subscenarios_market_price_profiles ( - market_price_profile_scenario_id INTEGER PRIMARY KEY AUTOINCREMENT, + market TEXT, + market_price_profile_scenario_id INTEGER, name VARCHAR(32), - description VARCHAR(128) + description VARCHAR(128), + PRIMARY KEY (market, market_price_profile_scenario_id) ); DROP TABLE IF EXISTS inputs_market_price_profiles; CREATE TABLE inputs_market_price_profiles ( + market TEXT, market_price_profile_scenario_id INTEGER, weather_iteration INTEGER NOT NULL, hydro_iteration INTEGER NOT NULL, stage_id INTEGER, timepoint INTEGER, - market VARCHAR(32), market_price FLOAT, - PRIMARY KEY (market_price_profile_scenario_id, weather_iteration, - hydro_iteration, stage_id, timepoint, - market) + PRIMARY KEY (market, market_price_profile_scenario_id, weather_iteration, + hydro_iteration, stage_id, timepoint) ); DROP TABLE IF EXISTS subscenarios_market_volume; diff --git a/gridpath/system/markets/prices.py b/gridpath/system/markets/prices.py index 663e22643..09adac5d9 100644 --- a/gridpath/system/markets/prices.py +++ b/gridpath/system/markets/prices.py @@ -84,50 +84,72 @@ def get_inputs_from_database( """ c = conn.cursor() - - (varies_by_weather_iteration, varies_by_hydro_iteration) = c.execute( + market_list = c.execute( f""" - SELECT varies_by_weather_iteration, varies_by_hydro_iteration + SELECT market, + market_price_profile_scenario_id, + varies_by_weather_iteration, + varies_by_hydro_iteration FROM inputs_market_prices - WHERE market_price_scenario_id = {subscenarios.MARKET_PRICE_SCENARIO_ID} - """ - ).fetchone() - - weather_iteration_to_use = weather_iteration if varies_by_weather_iteration else 0 - hydro_iteration_to_use = hydro_iteration if varies_by_hydro_iteration else 0 - - c1 = conn.cursor() - prices = c1.execute( - f""" - SELECT market, timepoint, market_price + WHERE market_price_scenario_id = {subscenarios.MARKET_PRICE_SCENARIO_ID} -- Get prices for included markets only - FROM ( + AND market in ( SELECT market FROM inputs_geography_markets WHERE market_scenario_id = {subscenarios.MARKET_SCENARIO_ID} - ) as market_tbl - -- Get prices for included timepoints only - CROSS JOIN ( - SELECT stage_id, timepoint from inputs_temporal - WHERE temporal_scenario_id = {subscenarios.TEMPORAL_SCENARIO_ID} - AND subproblem_id = {subproblem} - AND stage_id = {stage} - ) as tmp_tbl - LEFT OUTER JOIN ( - SELECT market, stage_id, timepoint, market_price - FROM inputs_market_price_profiles - WHERE market_price_profile_scenario_id = ( - SELECT market_price_profile_scenario_id - FROM inputs_market_prices - WHERE market_price_scenario_id = {subscenarios.MARKET_PRICE_SCENARIO_ID} - AND hydro_iteration = {hydro_iteration_to_use} - AND weather_iteration = {weather_iteration_to_use} - ) - ) as price_tbl - USING (market, stage_id, timepoint) - ; + ) """ - ) + ).fetchall() + + # Loop over the markets for the final query since prices don't all vary + # by the same iteration types + n_markets = len(market_list) + query_all = str() + n = 1 + for ( + market, + market_price_profile_scenario_id, + varies_by_weather_iteration, + varies_by_hydro_iteration, + ) in market_list: + union_str = "UNION" if n < n_markets else "" + + weather_iteration_to_use = ( + weather_iteration if varies_by_weather_iteration else 0 + ) + hydro_iteration_to_use = hydro_iteration if varies_by_hydro_iteration else 0 + + query_market = f""" + -- Select market name explicitly here to print even if prices + -- are not found for the relevant timepoints + SELECT '{market}' AS market, timepoint, market_price + -- Get prices for scenario's timepoints only + -- Note prices are required for all timepoints, so don't remove + -- the LEFT OUTER JOIN + FROM ( + SELECT stage_id, timepoint + FROM inputs_temporal + WHERE temporal_scenario_id = {subscenarios.TEMPORAL_SCENARIO_ID} + AND subproblem_id = {subproblem} + AND stage_id = {stage} + ) as tmp_tbl + LEFT OUTER JOIN ( + SELECT market, stage_id, timepoint, market_price + FROM inputs_market_price_profiles + WHERE market = '{market}' + AND market_price_profile_scenario_id = {market_price_profile_scenario_id} + AND hydro_iteration = {hydro_iteration_to_use} + AND weather_iteration = {weather_iteration_to_use} + ) as prices_tbl + USING (stage_id, timepoint) + {union_str} + """ + + query_all += query_market + n += 1 + + c1 = conn.cursor() + prices = c1.execute(query_all) return prices