diff --git a/src/fetch/orderbook.py b/src/fetch/orderbook.py index 0387e8a1..56f793e2 100644 --- a/src/fetch/orderbook.py +++ b/src/fetch/orderbook.py @@ -59,6 +59,17 @@ def _query_both_dbs( prod = cls._read_query_for_env(query, OrderbookEnv.PROD, data_types) return barn, prod + @classmethod + def _query_both_dbs_edit( + cls, + query_prod: str, + query_barn: str, + data_types: Optional[dict[str, str]] = None, + ) -> tuple[DataFrame, DataFrame]: + barn = cls._read_query_for_env(query_barn, OrderbookEnv.BARN, data_types) + prod = cls._read_query_for_env(query_prod, OrderbookEnv.PROD, data_types) + return barn, prod + @classmethod def get_latest_block(cls) -> int: """ @@ -78,13 +89,20 @@ def get_order_rewards(cls, block_range: BlockRange) -> DataFrame: """ Fetches and validates Order Reward DataFrame as concatenation from Prod and Staging DB """ - cow_reward_query = ( + cow_reward_query_prod = ( open_query("orderbook/order_rewards.sql") .replace("{{start_block}}", str(block_range.block_from)) .replace("{{end_block}}", str(block_range.block_to)) ) + cow_reward_query_barn = ( + open_query("orderbook/barn_order_rewards.sql") + .replace("{{start_block}}", str(block_range.block_from)) + .replace("{{end_block}}", str(block_range.block_to)) + ) data_types = {"block_number": "int64", "amount": "float64"} - barn, prod = cls._query_both_dbs(cow_reward_query, data_types) + barn, prod = cls._query_both_dbs_edit( + cow_reward_query_prod, cow_reward_query_barn, data_types + ) # Solvers do not appear in both environments! assert set(prod.solver).isdisjoint(set(barn.solver)), "solver overlap!" @@ -101,7 +119,18 @@ def get_batch_rewards(cls, block_range: BlockRange) -> DataFrame: """ Fetches and validates Batch Rewards DataFrame as concatenation from Prod and Staging DB """ - cow_reward_query = ( + cow_reward_query_prod = ( + open_query("orderbook/batch_rewards.sql") + .replace("{{start_block}}", str(block_range.block_from)) + .replace("{{end_block}}", str(block_range.block_to)) + .replace( + "{{EPSILON_LOWER}}", "10000000000000000" + ) # lower ETH cap for payment (in WEI) + .replace( + "{{EPSILON_UPPER}}", "12000000000000000" + ) # upper ETH cap for payment (in WEI) + ) + cow_reward_query_barn = ( open_query("orderbook/batch_rewards.sql") .replace("{{start_block}}", str(block_range.block_from)) .replace("{{end_block}}", str(block_range.block_to)) @@ -118,7 +147,9 @@ def get_batch_rewards(cls, block_range: BlockRange) -> DataFrame: "block_number": "Int64", "block_deadline": "int64", } - barn, prod = cls._query_both_dbs(cow_reward_query, data_types) + barn, prod = cls._query_both_dbs_edit( + cow_reward_query_prod, cow_reward_query_barn, data_types + ) # Solvers do not appear in both environments! assert set(prod.solver).isdisjoint(set(barn.solver)), "solver overlap!" diff --git a/src/sql/orderbook/barn_batch_rewards.sql b/src/sql/orderbook/barn_batch_rewards.sql new file mode 100644 index 00000000..11e1aa99 --- /dev/null +++ b/src/sql/orderbook/barn_batch_rewards.sql @@ -0,0 +1,241 @@ +WITH observed_settlements AS ( + SELECT + -- settlement + tx_hash, + solver, + s.block_number, + -- settlement_observations + effective_gas_price * gas_used AS execution_cost, + surplus, + fee, + s.auction_id + FROM + settlement_observations so + JOIN settlements s ON s.block_number = so.block_number + AND s.log_index = so.log_index + JOIN settlement_scores ss ON s.auction_id = ss.auction_id + WHERE + ss.block_deadline > {{start_block}} + AND ss.block_deadline <= {{end_block}} +), +auction_participation as ( + SELECT + ss.auction_id, + array_agg( + concat('0x', encode(participant, 'hex')) + ORDER BY + participant + ) as participating_solvers + FROM + auction_participants + JOIN settlement_scores ss ON auction_participants.auction_id = ss.auction_id + WHERE + block_deadline > {{start_block}} + AND block_deadline <= {{end_block}} + GROUP BY + ss.auction_id +), +-- protocol fees: +order_surplus AS ( + SELECT + ss.winner as solver, + s.auction_id, + s.tx_hash, + t.order_uid, + o.sell_token, + o.buy_token, + t.sell_amount, -- the total amount the user sends + t.buy_amount, -- the total amount the user receives + oe.surplus_fee as observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price + o.kind, + CASE + WHEN o.kind = 'sell' THEN t.buy_amount - t.sell_amount * o.buy_amount / (o.sell_amount + o.fee_amount) + WHEN o.kind = 'buy' THEN t.buy_amount * (o.sell_amount + o.fee_amount) / o.buy_amount - t.sell_amount + END AS surplus, + CASE + WHEN o.kind = 'sell' THEN o.buy_token + WHEN o.kind = 'buy' THEN o.sell_token + END AS surplus_token + FROM + settlements s + JOIN settlement_scores ss -- contains block_deadline + ON s.auction_id = ss.auction_id + JOIN trades t -- contains traded amounts + ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block + JOIN orders o -- contains tokens and limit amounts + ON t.order_uid = o.uid + JOIN order_execution oe -- contains surplus fee + ON t.order_uid = oe.order_uid + AND s.auction_id = oe.auction_id + WHERE + ss.block_deadline > {{start_block}} + AND ss.block_deadline <= {{end_block}} +), +order_protocol_fee AS ( + SELECT + os.auction_id, + os.solver, + os.tx_hash, + os.sell_amount, + os.buy_amount, + os.sell_token, + os.observed_fee, + os.surplus, + os.surplus_token, + CASE + WHEN fp.kind = 'surplus' THEN CASE + WHEN os.kind = 'sell' THEN + -- We assume that the case surplus_factor != 1 always. In + -- that case reconstructing the protocol fee would be + -- impossible anyways. This query will return a division by + -- zero error in that case. + LEAST( + fp.surplus_max_volume_factor * os.sell_amount * os.buy_amount / (os.sell_amount - os.observed_fee), -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus + ) + WHEN os.kind = 'buy' THEN LEAST( + fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus + ) + END + WHEN fp.kind = 'volume' THEN fp.volume_factor / (1 + fp.volume_factor) * os.sell_amount + END AS protocol_fee, + CASE + WHEN fp.kind = 'surplus' THEN os.surplus_token + WHEN fp.kind = 'volume' THEN os.sell_token + END AS protocol_fee_token + FROM + order_surplus os + JOIN fee_policies fp -- contains protocol fee policy + ON os.auction_id = fp.auction_id + AND os.order_uid = fp.order_uid +), +order_protocol_fee_prices AS ( + SELECT + opf.solver, + opf.tx_hash, + opf.surplus, + opf.protocol_fee, + CASE + WHEN opf.sell_token != opf.protocol_fee_token THEN (opf.sell_amount - opf.observed_fee) / opf.buy_amount * opf.protocol_fee + ELSE opf.protocol_fee + END AS network_fee_correction, + opf.sell_token as network_fee_token, + ap_surplus.price / pow(10, 18) as surplus_token_price, + ap_protocol.price / pow(10, 18) as protocol_fee_token_price, + ap_sell.price / pow(10, 18) as network_fee_token_price + FROM + order_protocol_fee opf + JOIN auction_prices ap_sell -- contains price: sell token + ON opf.auction_id = ap_sell.auction_id + AND opf.sell_token = ap_sell.token + JOIN auction_prices ap_surplus -- contains price: surplus token + ON opf.auction_id = ap_surplus.auction_id + AND opf.surplus_token = ap_surplus.token + JOIN auction_prices ap_protocol -- contains price: protocol fee token + ON opf.auction_id = ap_protocol.auction_id + AND opf.protocol_fee_token = ap_protocol.token +), +batch_protocol_fees AS ( + SELECT + solver, + tx_hash, + -- sum(surplus * surplus_token_price) as surplus, + sum(protocol_fee * protocol_fee_token_price) as protocol_fee, + sum(network_fee_correction * network_fee_token_price) as network_fee_correction + FROM + order_protocol_fee_prices + group by + solver, + tx_hash +), +reward_data AS ( + SELECT + -- observations + os.tx_hash, + ss.auction_id, + -- TODO - Assuming that `solver == winner` when both not null + -- We will need to monitor that `solver == winner`! + coalesce(os.solver, winner) as solver, + block_number as settlement_block, + block_deadline, + case + when block_number is not null + and block_number > block_deadline then 0 + else coalesce(execution_cost, 0) + end as execution_cost, + case + when block_number is not null + and block_number > block_deadline then 0 + else coalesce(surplus, 0) + end as surplus, + case + when block_number is not null + and block_number > block_deadline then 0 + else coalesce(fee, 0) + end as fee, + -- scores + winning_score, + reference_score, + -- auction_participation + participating_solvers, + -- protocol_fees + coalesce(cast(protocol_fee as numeric(78, 0)), 0) as protocol_fee, + coalesce( + cast(network_fee_correction as numeric(78, 0)), + 0 + ) as network_fee_correction + FROM + settlement_scores ss + -- If there are reported scores, + -- there will always be a record of auction participants + JOIN auction_participation ap ON ss.auction_id = ap.auction_id + -- outer joins made in order to capture non-existent settlements. + LEFT OUTER JOIN observed_settlements os ON os.auction_id = ss.auction_id + LEFT OUTER JOIN batch_protocol_fees bpf ON bpf.tx_hash = os.tx_hash +), +reward_per_auction as ( + SELECT + tx_hash, + auction_id, + settlement_block, + block_deadline, + solver, + execution_cost, + surplus, + protocol_fee, -- the protocol fee + fee - network_fee_correction as network_fee, -- the network fee + surplus + protocol_fee + fee - network_fee_correction - reference_score as uncapped_payment_eth, + -- Capped Reward = CLAMP_[-E, E + exec_cost](uncapped_reward_eth) + LEAST( + GREATEST( + - {{EPSILON_LOWER}}, + surplus + protocol_fee + fee - network_fee_correction - reference_score + ), + {{EPSILON_UPPER}} + execution_cost + ) as capped_payment, + winning_score, + reference_score, + participating_solvers as participating_solvers + FROM + reward_data +) +SELECT + settlement_block as block_number, + block_deadline, + case + when tx_hash is NULL then NULL + else concat('0x', encode(tx_hash, 'hex')) + end as tx_hash, + concat('0x', encode(solver, 'hex')) as solver, + execution_cost :: text as execution_cost, + surplus :: text as surplus, + protocol_fee :: text as protocol_fee, + network_fee :: text as network_fee, + uncapped_payment_eth :: text as uncapped_payment_eth, + capped_payment :: text as capped_payment, + winning_score :: text as winning_score, + reference_score :: text as reference_score, + participating_solvers +FROM + reward_per_auction diff --git a/src/sql/orderbook/barn_order_rewards.sql b/src/sql/orderbook/barn_order_rewards.sql new file mode 100644 index 00000000..732c8e29 --- /dev/null +++ b/src/sql/orderbook/barn_order_rewards.sql @@ -0,0 +1,171 @@ +with trade_hashes as ( + SELECT + settlement.solver, + t.block_number as block_number, + order_uid, + fee_amount, + settlement.tx_hash, + auction_id + FROM + trades t + LEFT OUTER JOIN LATERAL ( + SELECT + tx_hash, + solver, + tx_nonce, + tx_from, + auction_id, + block_number, + log_index + FROM + settlements s + WHERE + s.block_number = t.block_number + AND s.log_index > t.log_index + ORDER BY + s.log_index ASC + LIMIT + 1 + ) AS settlement ON true + join settlement_observations so on settlement.block_number = so.block_number + and settlement.log_index = so.log_index + where + settlement.block_number > {{start_block}} + and settlement.block_number <= {{end_block}} +), +order_surplus AS ( + SELECT + s.auction_id, + t.order_uid, + o.sell_token, + o.buy_token, + t.sell_amount, -- the total amount the user sends + t.buy_amount, -- the total amount the user receives + oe.surplus_fee as observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price + o.kind, + CASE + WHEN o.kind = 'sell' THEN t.buy_amount - t.sell_amount * o.buy_amount / (o.sell_amount + o.fee_amount) + WHEN o.kind = 'buy' THEN t.buy_amount * (o.sell_amount + o.fee_amount) / o.buy_amount - t.sell_amount + END AS surplus, + CASE + WHEN o.kind = 'sell' THEN o.buy_token + WHEN o.kind = 'buy' THEN o.sell_token + END AS surplus_token + FROM + settlements s -- links block_number and log_index to tx_from and tx_nonce + JOIN settlement_scores ss -- contains block_deadline + ON s.auction_id = ss.auction_id + JOIN trades t -- contains traded amounts + ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block + JOIN orders o -- contains tokens and limit amounts + ON t.order_uid = o.uid + JOIN order_execution oe -- contains surplus fee + ON t.order_uid = oe.order_uid + AND s.auction_id = oe.auction_id + WHERE + s.block_number > {{start_block}} + AND s.block_number <= {{end_block}} +), +order_protocol_fee AS ( + SELECT + os.auction_id, + os.order_uid, + os.sell_amount, + os.buy_amount, + os.sell_token, + os.observed_fee, + os.surplus, + os.surplus_token, + CASE + WHEN fp.kind = 'surplus' THEN CASE + WHEN os.kind = 'sell' THEN + -- We assume that the case surplus_factor != 1 always. In + -- that case reconstructing the protocol fee would be + -- impossible anyways. This query will return a division by + -- zero error in that case. + LEAST( + fp.surplus_max_volume_factor * os.sell_amount * os.buy_amount / (os.sell_amount - os.observed_fee), -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus + ) + WHEN os.kind = 'buy' THEN LEAST( + fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus + ) + END + WHEN fp.kind = 'volume' THEN fp.volume_factor / (1 - fp.volume_factor) * os.sell_amount + END AS protocol_fee, + CASE + WHEN fp.kind = 'surplus' THEN os.surplus_token + WHEN fp.kind = 'volume' THEN os.sell_token + END AS protocol_fee_token + FROM + order_surplus os + JOIN fee_policies fp -- contains protocol fee policy + ON os.auction_id = fp.auction_id + AND os.order_uid = fp.order_uid +), +order_protocol_fee_prices AS ( + SELECT + opf.order_uid, + opf.auction_id, + opf.protocol_fee, + opf.protocol_fee_token, + ap.price / pow(10, 18) as protocol_fee_native_price + FROM + order_protocol_fee opf + JOIN auction_prices ap -- contains price: protocol fee token + ON opf.auction_id = ap.auction_id + AND opf.protocol_fee_token = ap.token +), +winning_quotes as ( + SELECT + concat('0x', encode(oq.solver, 'hex')) as quote_solver, + oq.order_uid + FROM + trades t + INNER JOIN orders o ON order_uid = uid + JOIN order_quotes oq ON t.order_uid = oq.order_uid + WHERE + ( + o.class = 'market' + OR ( + o.kind = 'sell' + AND ( + oq.sell_amount - oq.gas_amount * oq.gas_price / oq.sell_token_price + ) * oq.buy_amount >= o.buy_amount * oq.sell_amount + ) + OR ( + o.kind = 'buy' + AND o.sell_amount >= oq.sell_amount + oq.gas_amount * oq.gas_price / oq.sell_token_price + ) + ) + AND o.partially_fillable = 'f' -- the code above might fail for partially fillable orders + AND t.block_number > {{start_block}} + AND t.block_number <= {{end_block}} + AND oq.solver != '\x0000000000000000000000000000000000000000' +) -- Most efficient column order for sorting would be having tx_hash or order_uid first +select + trade_hashes.block_number as block_number, + concat('0x', encode(trade_hashes.order_uid, 'hex')) as order_uid, + concat('0x', encode(trade_hashes.solver, 'hex')) as solver, + quote_solver, + concat('0x', encode(tx_hash, 'hex')) as tx_hash, + coalesce(surplus_fee, 0) :: text as surplus_fee, + coalesce(reward, 0.0) as amount, + coalesce(cast(protocol_fee as numeric(78, 0)), 0) :: text as protocol_fee, + CASE + WHEN protocol_fee_token is not NULL THEN concat('0x', encode(protocol_fee_token, 'hex')) + END as protocol_fee_token, + coalesce(protocol_fee_native_price, 0.0) as protocol_fee_native_price, + cast(oq.sell_amount as numeric(78, 0)) :: text as quote_sell_amount, + cast(oq.buy_amount as numeric(78, 0)) :: text as quote_buy_amount, + oq.gas_amount * oq.gas_price as quote_gas_cost, + oq.sell_token_price as quote_sell_token_price +from + trade_hashes + left outer join order_execution o on trade_hashes.order_uid = o.order_uid + and trade_hashes.auction_id = o.auction_id + left outer join winning_quotes wq on trade_hashes.order_uid = wq.order_uid + left outer join order_protocol_fee_prices opfp on trade_hashes.order_uid = opfp.order_uid + and trade_hashes.auction_id = opfp.auction_id + left outer join order_quotes oq on trade_hashes.order_uid = oq.order_uid \ No newline at end of file