Merge branch 'main' into cip-38

src/fetch/orderbook.py

@@ -53,10 +53,13 @@ def _read_query_for_env(
 
     @classmethod
     def _query_both_dbs(
-        cls, query: str, data_types: Optional[dict[str, str]] = None
+        cls,
+        query_prod: str,
+        query_barn: str,
+        data_types: Optional[dict[str, str]] = None,
     ) -> tuple[DataFrame, DataFrame]:
-        barn = cls._read_query_for_env(query, OrderbookEnv.BARN, data_types)
-        prod = cls._read_query_for_env(query, OrderbookEnv.PROD, data_types)
+        barn = cls._read_query_for_env(query_barn, OrderbookEnv.BARN, data_types)
+        prod = cls._read_query_for_env(query_prod, OrderbookEnv.PROD, data_types)
         return barn, prod
 
     @classmethod
@@ -65,9 +68,8 @@ def get_latest_block(cls) -> int:
         Fetches the latest mutually synced block from orderbook databases (with REORG protection)
         """
         data_types = {"latest": "int64"}
-        barn, prod = cls._query_both_dbs(
-            open_query("orderbook/latest_block.sql"), data_types
-        )
+        query_barn_prod = open_query("orderbook/latest_block.sql")
+        barn, prod = cls._query_both_dbs(query_barn_prod, query_barn_prod, data_types)
         assert len(barn) == 1 == len(prod), "Expecting single record"
         return (
             max(int(barn["latest"][0]), int(prod["latest"][0])) - MAX_PROCESSING_DELAY
@@ -78,13 +80,20 @@ def get_order_rewards(cls, block_range: BlockRange) -> DataFrame:
         """
         Fetches and validates Order Reward DataFrame as concatenation from Prod and Staging DB
         """
-        cow_reward_query = (
-            open_query("orderbook/order_rewards.sql")
+        cow_reward_query_prod = (
+            open_query("orderbook/prod_order_rewards.sql")
+            .replace("{{start_block}}", str(block_range.block_from))
+            .replace("{{end_block}}", str(block_range.block_to))
+        )
+        cow_reward_query_barn = (
+            open_query("orderbook/barn_order_rewards.sql")
             .replace("{{start_block}}", str(block_range.block_from))
             .replace("{{end_block}}", str(block_range.block_to))
         )
         data_types = {"block_number": "int64", "amount": "float64"}
-        barn, prod = cls._query_both_dbs(cow_reward_query, data_types)
+        barn, prod = cls._query_both_dbs(
+            cow_reward_query_prod, cow_reward_query_barn, data_types
+        )
 
         # Solvers do not appear in both environments!
         assert set(prod.solver).isdisjoint(set(barn.solver)), "solver overlap!"
@@ -101,8 +110,19 @@ def get_batch_rewards(cls, block_range: BlockRange) -> DataFrame:
         """
         Fetches and validates Batch Rewards DataFrame as concatenation from Prod and Staging DB
         """
-        cow_reward_query = (
-            open_query("orderbook/batch_rewards.sql")
+        cow_reward_query_prod = (
+            open_query("orderbook/prod_batch_rewards.sql")
+            .replace("{{start_block}}", str(block_range.block_from))
+            .replace("{{end_block}}", str(block_range.block_to))
+            .replace(
+                "{{EPSILON_LOWER}}", "10000000000000000"
+            )  # lower ETH cap for payment (in WEI)
+            .replace(
+                "{{EPSILON_UPPER}}", "12000000000000000"
+            )  # upper ETH cap for payment (in WEI)
+        )
+        cow_reward_query_barn = (
+            open_query("orderbook/barn_batch_rewards.sql")
             .replace("{{start_block}}", str(block_range.block_from))
             .replace("{{end_block}}", str(block_range.block_to))
             .replace(
@@ -118,7 +138,9 @@ def get_batch_rewards(cls, block_range: BlockRange) -> DataFrame:
             "block_number": "Int64",
             "block_deadline": "int64",
         }
-        barn, prod = cls._query_both_dbs(cow_reward_query, data_types)
+        barn, prod = cls._query_both_dbs(
+            cow_reward_query_prod, cow_reward_query_barn, data_types
+        )
 
         # Solvers do not appear in both environments!
         assert set(prod.solver).isdisjoint(set(barn.solver)), "solver overlap!"

src/sql/orderbook/batch_rewards.sql → src/sql/orderbook/barn_batch_rewards.sql

@@ -90,22 +90,22 @@ order_protocol_fee AS (
                 -- impossible anyways. This query will return a division by
                 -- zero error in that case.
                 LEAST(
-                    fp.max_volume_factor * os.sell_amount * os.buy_amount / (os.sell_amount - os.observed_fee),
-                    -- at most charge a fraction of volume
+                    fp.surplus_max_volume_factor / (1 - fp.surplus_max_volume_factor) * os.buy_amount, -- at most charge a fraction of volume
                     fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus
                 )
                 WHEN os.kind = 'buy' THEN LEAST(
-                    fp.max_volume_factor / (1 + fp.max_volume_factor) * os.sell_amount,
-                    -- at most charge a fraction of volume
+                    fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume
                     fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus
                 )
             END
-            WHEN fp.kind = 'volume' THEN fp.volume_factor / (1 + fp.volume_factor) * os.sell_amount
+            WHEN fp.kind = 'volume' THEN CASE
+                WHEN os.kind = 'sell' THEN
+                    fp.volume_factor / (1 - fp.volume_factor) * os.buy_amount
+                WHEN os.kind = 'buy' THEN
+                    fp.volume_factor / (1 + fp.volume_factor) * os.sell_amount
+            END
         END AS protocol_fee,
-        CASE
-            WHEN fp.kind = 'surplus' THEN os.surplus_token
-            WHEN fp.kind = 'volume' THEN os.sell_token
-        END AS protocol_fee_token
+        os.surplus_token as protocol_fee_token
     FROM
         order_surplus os
         JOIN fee_policies fp -- contains protocol fee policy
@@ -164,17 +164,17 @@ reward_data AS (
         case
             when block_number is not null
             and block_number > block_deadline then 0
-            else coalesce(execution_cost, 0) -- if block_number is null, execution cost is 0
+            else coalesce(execution_cost, 0)
         end as execution_cost,
         case
             when block_number is not null
             and block_number > block_deadline then 0
-            else coalesce(surplus, 0) -- if block_number is null, surplus is 0
+            else coalesce(surplus, 0)
         end as surplus,
         case
             when block_number is not null
             and block_number > block_deadline then 0
-            else coalesce(fee, 0) -- if block_number is null, fee is 0
+            else coalesce(fee, 0)
         end as fee,
         -- scores
         winning_score,
@@ -188,9 +188,11 @@ reward_data AS (
             0
         ) as network_fee_correction
     FROM
-        settlement_scores ss -- If there are reported scores,
+        settlement_scores ss 
+        -- If there are reported scores,
         -- there will always be a record of auction participants
-        JOIN auction_participation ap ON ss.auction_id = ap.auction_id -- outer joins made in order to capture non-existent settlements.
+        JOIN auction_participation ap ON ss.auction_id = ap.auction_id
+        -- outer joins made in order to capture non-existent settlements.
         LEFT OUTER JOIN observed_settlements os ON os.auction_id = ss.auction_id
         LEFT OUTER JOIN batch_protocol_fees bpf ON bpf.tx_hash = os.tx_hash
 ),
@@ -205,14 +207,14 @@ reward_per_auction as (
         surplus,
         protocol_fee, -- the protocol fee
         fee - network_fee_correction as network_fee, -- the network fee
-        surplus + protocol_fee - reference_score as uncapped_payment,
+        surplus + protocol_fee + fee - network_fee_correction - reference_score as uncapped_payment_eth,
         -- Capped Reward = CLAMP_[-E, E + exec_cost](uncapped_reward_eth)
         LEAST(
             GREATEST(
                 - {{EPSILON_LOWER}},
-                surplus + protocol_fee - reference_score
+                surplus + protocol_fee + fee - network_fee_correction - reference_score
             ),
-            {{EPSILON_UPPER}}
+            {{EPSILON_UPPER}} + execution_cost
         ) as capped_payment,
         winning_score,
         reference_score,
@@ -232,7 +234,7 @@ SELECT
     surplus :: text as surplus,
     protocol_fee :: text as protocol_fee,
     network_fee :: text as network_fee,
-    uncapped_payment_eth :: text as uncapped_payment,
+    uncapped_payment_eth :: text as uncapped_payment_eth,
     capped_payment :: text as capped_payment,
     winning_score :: text as winning_score,
     reference_score :: text as reference_score,

src/sql/orderbook/order_rewards.sql → src/sql/orderbook/barn_order_rewards.sql

@@ -84,11 +84,11 @@ order_protocol_fee AS (
                 -- impossible anyways. This query will return a division by
                 -- zero error in that case.
                 LEAST(
-                    fp.max_volume_factor * os.sell_amount * os.buy_amount / (os.sell_amount - os.observed_fee), -- at most charge a fraction of volume
+                    fp.surplus_max_volume_factor * os.sell_amount * os.buy_amount / (os.sell_amount - os.observed_fee), -- at most charge a fraction of volume
                     fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus
                 )
                 WHEN os.kind = 'buy' THEN LEAST(
-                    fp.max_volume_factor / (1 + fp.max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume
+                    fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume
                     fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus
                 )
             END