Market prices by market and iteration type (#1181)

db/csvs_test_examples/csv_structure.csv

@@ -155,7 +155,7 @@ reliability/local_capacity/system_local_capacity_requirement,local_capacity,loca
 markets/geography_markets,markets,market_scenario_id,geography_markets,simple,,0,,,,,
 markets/load_zone_markets,markets,load_zone_market_scenario_id,load_zone_markets,simple,,0,,,,,
 markets/prices,markets,market_price_scenario_id,market_prices,simple,,0,,,,,
-markets/prices/price_profiles,markets,market_price_profile_scenario_id,market_price_profiles,simple,,0,,,,,
+markets/prices/price_profiles,markets,market_price_profile_scenario_id,market_price_profiles,simple,,1,market,,,market_price_scenario_id,market_prices
 markets/volume,markets,market_volume_scenario_id,market_volume,simple,,0,,,,,
 water/water_system_params,water,water_system_params_scenario_id,system_water_system_params,simple,,0,,,,,
 water/water_network,water,water_network_scenario_id,geography_water_network,simple,,0,,,,,

db/csvs_test_examples/markets/prices/1_base_prices.csv

@@ -1,2 +1,3 @@
-market_price_profile_scenario_id,varies_by_weather_iteration,varies_by_hydro_iteration
-1,0,0
+market,market_price_profile_scenario_id,varies_by_weather_iteration,varies_by_hydro_iteration
+Market_Hub,1,0,0
+Market_Hub2,1,0,0

db/csvs_test_examples/markets/prices/price_profiles/Market_Hub-1-base_price_profiles.csv

@@ -0,0 +1,7 @@
+weather_iteration,hydro_iteration,stage_id,timepoint,market_price
+0,0,1,20200101,10
+0,0,1,20200102,-10
+0,0,1,20200201,10
+0,0,1,20200202,-10
+0,0,1,20200301,10
+0,0,1,20200302,-10

db/csvs_test_examples/markets/prices/price_profiles/Market_Hub2-1-base_price_profiles.csv

@@ -0,0 +1,13 @@
+weather_iteration,hydro_iteration,stage_id,timepoint,market_price
+0,0,1,20200101,1
+0,0,1,20200102,2
+0,0,1,20200201,3
+0,0,1,20200202,2
+0,0,1,20200301,1
+0,0,1,20200302,2
+0,0,2,20200101,3
+0,0,2,20200102,2
+0,0,2,20200201,1
+0,0,2,20200202,2
+0,0,2,20200301,3
+0,0,2,20200302,2

db/db_schema.sql

@@ -933,35 +933,38 @@ DROP TABLE IF EXISTS inputs_market_prices;
 CREATE TABLE inputs_market_prices
 (
     market_price_scenario_id         INTEGER,
+    market                           TEXT,
     market_price_profile_scenario_id INTEGER,
     varies_by_weather_iteration      INTEGER,
     varies_by_hydro_iteration        INTEGER,
-    PRIMARY KEY (market_price_scenario_id, market_price_profile_scenario_id),
+    PRIMARY KEY (market_price_scenario_id, market,
+                 market_price_profile_scenario_id),
     FOREIGN KEY (market_price_scenario_id) REFERENCES
         subscenarios_market_prices (market_price_scenario_id)
 );
 
 DROP TABLE IF EXISTS subscenarios_market_price_profiles;
 CREATE TABLE subscenarios_market_price_profiles
 (
-    market_price_profile_scenario_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    market                           TEXT,
+    market_price_profile_scenario_id INTEGER,
     name                             VARCHAR(32),
-    description                      VARCHAR(128)
+    description                      VARCHAR(128),
+    PRIMARY KEY (market, market_price_profile_scenario_id)
 );
 
 DROP TABLE IF EXISTS inputs_market_price_profiles;
 CREATE TABLE inputs_market_price_profiles
 (
+    market                           TEXT,
     market_price_profile_scenario_id INTEGER,
     weather_iteration                INTEGER NOT NULL,
     hydro_iteration                  INTEGER NOT NULL,
     stage_id                         INTEGER,
     timepoint                        INTEGER,
-    market                           VARCHAR(32),
     market_price                     FLOAT,
-    PRIMARY KEY (market_price_profile_scenario_id, weather_iteration,
-                 hydro_iteration, stage_id, timepoint,
-                 market)
+    PRIMARY KEY (market, market_price_profile_scenario_id, weather_iteration,
+                 hydro_iteration, stage_id, timepoint)
 );
 
 DROP TABLE IF EXISTS subscenarios_market_volume;

gridpath/system/markets/prices.py

@@ -84,50 +84,72 @@ def get_inputs_from_database(
     """
 
     c = conn.cursor()
-
-    (varies_by_weather_iteration, varies_by_hydro_iteration) = c.execute(
+    market_list = c.execute(
         f"""
-        SELECT varies_by_weather_iteration, varies_by_hydro_iteration
+        SELECT market, 
+        market_price_profile_scenario_id,
+        varies_by_weather_iteration, 
+        varies_by_hydro_iteration
         FROM inputs_market_prices
-        WHERE market_price_scenario_id = {subscenarios.MARKET_PRICE_SCENARIO_ID} 
-        """
-    ).fetchone()
-
-    weather_iteration_to_use = weather_iteration if varies_by_weather_iteration else 0
-    hydro_iteration_to_use = hydro_iteration if varies_by_hydro_iteration else 0
-
-    c1 = conn.cursor()
-    prices = c1.execute(
-        f"""
-        SELECT market, timepoint, market_price
+        WHERE market_price_scenario_id = {subscenarios.MARKET_PRICE_SCENARIO_ID}
         -- Get prices for included markets only
-        FROM (
+        AND market in (
             SELECT market
             FROM inputs_geography_markets
             WHERE market_scenario_id = {subscenarios.MARKET_SCENARIO_ID}
-        ) as market_tbl
-        -- Get prices for included timepoints only
-        CROSS JOIN (
-            SELECT stage_id, timepoint from inputs_temporal
-            WHERE temporal_scenario_id = {subscenarios.TEMPORAL_SCENARIO_ID}
-            AND subproblem_id = {subproblem}
-            AND stage_id = {stage}
-        ) as tmp_tbl
-        LEFT OUTER JOIN (
-            SELECT market, stage_id, timepoint, market_price
-            FROM inputs_market_price_profiles
-            WHERE market_price_profile_scenario_id = (
-            SELECT market_price_profile_scenario_id
-            FROM inputs_market_prices
-            WHERE market_price_scenario_id = {subscenarios.MARKET_PRICE_SCENARIO_ID}
-            AND hydro_iteration = {hydro_iteration_to_use}
-            AND weather_iteration = {weather_iteration_to_use}
-            ) 
-        ) as price_tbl
-        USING (market, stage_id, timepoint)
-        ;
+        )
         """
-    )
+    ).fetchall()
+
+    # Loop over the markets for the final query since prices don't all vary
+    # by the same iteration types
+    n_markets = len(market_list)
+    query_all = str()
+    n = 1
+    for (
+        market,
+        market_price_profile_scenario_id,
+        varies_by_weather_iteration,
+        varies_by_hydro_iteration,
+    ) in market_list:
+        union_str = "UNION" if n < n_markets else ""
+
+        weather_iteration_to_use = (
+            weather_iteration if varies_by_weather_iteration else 0
+        )
+        hydro_iteration_to_use = hydro_iteration if varies_by_hydro_iteration else 0
+
+        query_market = f"""
+            -- Select market name explicitly here to print even if prices 
+            -- are not found for the relevant timepoints
+            SELECT '{market}' AS market, timepoint, market_price
+            -- Get prices for scenario's timepoints only
+            -- Note prices are required for all timepoints, so don't remove 
+            -- the LEFT OUTER JOIN
+            FROM (
+                SELECT stage_id, timepoint 
+                FROM inputs_temporal
+                WHERE temporal_scenario_id = {subscenarios.TEMPORAL_SCENARIO_ID}
+                AND subproblem_id = {subproblem}
+                AND stage_id = {stage}
+            ) as tmp_tbl
+            LEFT OUTER JOIN (
+                SELECT market, stage_id, timepoint, market_price
+                FROM inputs_market_price_profiles
+                WHERE market = '{market}'
+                AND market_price_profile_scenario_id = {market_price_profile_scenario_id}
+                AND hydro_iteration = {hydro_iteration_to_use}
+                AND weather_iteration = {weather_iteration_to_use}
+            ) as prices_tbl
+            USING (stage_id, timepoint)
+            {union_str}
+            """
+
+        query_all += query_market
+        n += 1
+
+    c1 = conn.cursor()
+    prices = c1.execute(query_all)
 
     return prices