From 9305c01b74c8fff851320926a85404b604f9c663 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Thu, 12 Dec 2024 10:22:53 +0100 Subject: [PATCH] Fix merging of data frames for payments (#453) This PR changes which columns are expected from dune results and restricts columns before merging. This is necessary since multiple queries now contain a `solver_name` field. In merging, the filed was renamed to `solver_name_x`, `solver_name_y`, ..., resulting in uncaught exceptions. This can be tested by running `python -m src.fetch.transfer_file --start 2024-11-26 --post-tx --dry-run --min-transfer-amount-wei "1000000000000000" --min-transfer-amount-cow-atoms "1000000000000000000" --ignore-slippage` with suitable environments set up for Gnosis and Arbitrum One. --------- Co-authored-by: Haris Angelidakis <64154020+harisang@users.noreply.github.com> --- src/fetch/payouts.py | 37 ++++++++++----------- tests/unit/test_payouts.py | 67 +++++++++++++++++++------------------- 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/src/fetch/payouts.py b/src/fetch/payouts.py index 4c44d01f..d83874f4 100644 --- a/src/fetch/payouts.py +++ b/src/fetch/payouts.py @@ -36,10 +36,9 @@ } SLIPPAGE_COLUMNS = { "solver", - "solver_name", "eth_slippage_wei", } -REWARD_TARGET_COLUMNS = {"solver", "reward_target", "pool_address"} +REWARD_TARGET_COLUMNS = {"solver", "solver_name", "reward_target", "pool_address"} SERVICE_FEE_COLUMNS = {"solver", "service_fee"} ADDITIONAL_PAYMENT_COLUMNS = {"buffer_accounting_target", "reward_token_address"} @@ -416,9 +415,12 @@ def construct_payout_dataframe( # 3. Merge the three dataframes (joining on solver) merged_df = ( - payment_df.merge(slippage_df, on=join_column, how="left") - .merge(reward_target_df, on=join_column, how="left") - .merge(service_fee_df, on=join_column, how="left") + payment_df[list(PAYMENT_COLUMNS)] + .merge(slippage_df[list(SLIPPAGE_COLUMNS)], on=join_column, how="left") + .merge( + reward_target_df[list(REWARD_TARGET_COLUMNS)], on=join_column, how="left" + ) + .merge(service_fee_df[list(SERVICE_FEE_COLUMNS)], on=join_column, how="left") ) # 4. Add slippage from fees to slippage @@ -510,11 +512,15 @@ def construct_payouts( quote_rewards_df, batch_rewards_df, on="solver", how="outer" ).fillna(0) - service_fee_df = pandas.DataFrame(dune.get_service_fee_status()) - service_fee_df["service_fee"] = [ - service_fee_flag * config.reward_config.service_fee_factor - for service_fee_flag in service_fee_df["service_fee"] - ] + service_fee_dune = dune.get_service_fee_status() + if service_fee_dune: + service_fee_df = pandas.DataFrame(service_fee_dune) + service_fee_df["service_fee"] = [ + service_fee_flag * config.reward_config.service_fee_factor + for service_fee_flag in service_fee_df["service_fee"] + ] + else: + service_fee_df = DataFrame(columns=["solver", "service_fee"]) vouches = dune.get_vouches() if vouches: @@ -525,15 +531,8 @@ def construct_payouts( ) # construct slippage df if ignore_slippage_flag or (not config.buffer_accounting_config.include_slippage): - slippage_df_temp = pandas.merge( - merged_df[["solver"]], - reward_target_df[["solver", "solver_name"]], - on="solver", - how="inner", - ) - slippage_df = slippage_df_temp.assign( - eth_slippage_wei=[0] * slippage_df_temp.shape[0] - ) + slippage_df = merged_df[["solver"]].copy() + slippage_df["eth_slippage_wei"] = [0] * slippage_df.shape[0] else: slippage_df = pandas.DataFrame(dune.get_period_slippage()) # TODO - After CIP-20 phased in, adapt query to return `solver` like all the others diff --git a/tests/unit/test_payouts.py b/tests/unit/test_payouts.py index 3b95e5e8..05e2c566 100644 --- a/tests/unit/test_payouts.py +++ b/tests/unit/test_payouts.py @@ -153,11 +153,9 @@ def test_validate_df_columns(self): "quote_reward_cow": [], } ) - legit_slippages = DataFrame( - {"solver": [], "solver_name": [], "eth_slippage_wei": []} - ) + legit_slippages = DataFrame({"solver": [], "eth_slippage_wei": []}) legit_reward_targets = DataFrame( - {"solver": [], "reward_target": [], "pool_address": []} + {"solver": [], "solver_name": [], "reward_target": [], "pool_address": []} ) legit_service_fees = DataFrame({"solver": [], "service_fee": []}) @@ -221,7 +219,6 @@ def test_construct_payouts(self): "solver": self.solvers[:3], # Note that one of the solvers did not appear, # in this list (we are testing the left join) - "solver_name": ["S_1", "S_2", "S_3"], "eth_slippage_wei": [1, 0, -1], } ) @@ -229,6 +226,7 @@ def test_construct_payouts(self): reward_targets = DataFrame( { "solver": self.solvers, + "solver_name": ["S_1", "S_2", "S_3", "S_4"], "reward_target": self.reward_targets, "pool_address": self.pool_addresses, } @@ -247,46 +245,55 @@ def test_construct_payouts(self): ) expected = DataFrame( { - "solver": self.solvers, - "num_quotes": self.num_quotes, - "primary_reward_eth": [600000000000000.0, 1.2e16, -1e16, 0.0], - "protocol_fee_eth": [ - 1000000000000000.0, - 2000000000000000.0, - 0.0, - 0.0, + "buffer_accounting_target": [ + "0x0000000000000000000000000000000000000005", + str(self.solvers[1]), + str(self.solvers[2]), + str(self.solvers[3]), ], + "eth_slippage_wei": [2000000000000001.0, 4000000000000000.0, -1.0, 0.0], "network_fee_eth": [ 2000000000000000.0, 4000000000000000.0, 0.0, 0.0, ], + "pool_address": [ + str(self.config.reward_config.cow_bonding_pool), + "0x0000000000000000000000000000000000000010", + "0x0000000000000000000000000000000000000011", + "0x0000000000000000000000000000000000000012", + ], "primary_reward_cow": [ 600000000000000000.0, 12000000000000000000.0, -10000000000000000000.0, 0.0, ], + "primary_reward_eth": [600000000000000.0, 1.2e16, -1e16, 0.0], + "protocol_fee_eth": [ + 1000000000000000.0, + 2000000000000000.0, + 0.0, + 0.0, + ], "quote_reward_cow": [ 0.00000, 0.00000, 6000000000000000000.00000, 12000000000000000000.00000, ], - "solver_name": ["S_1", "S_2", "S_3", None], - "eth_slippage_wei": [2000000000000001.0, 4000000000000000.0, -1.0, 0.0], "reward_target": [ "0x0000000000000000000000000000000000000005", "0x0000000000000000000000000000000000000006", "0x0000000000000000000000000000000000000007", "0x0000000000000000000000000000000000000008", ], - "pool_address": [ - str(self.config.reward_config.cow_bonding_pool), - "0x0000000000000000000000000000000000000010", - "0x0000000000000000000000000000000000000011", - "0x0000000000000000000000000000000000000012", + "reward_token_address": [ + str(self.config.reward_config.reward_token_address), + str(self.config.reward_config.reward_token_address), + str(self.config.reward_config.reward_token_address), + str(self.config.reward_config.reward_token_address), ], "service_fee": [ Fraction(0, 100), @@ -294,22 +301,16 @@ def test_construct_payouts(self): Fraction(0, 100), Fraction(15, 100), ], - "buffer_accounting_target": [ - "0x0000000000000000000000000000000000000005", - str(self.solvers[1]), - str(self.solvers[2]), - str(self.solvers[3]), - ], - "reward_token_address": [ - str(self.config.reward_config.reward_token_address), - str(self.config.reward_config.reward_token_address), - str(self.config.reward_config.reward_token_address), - str(self.config.reward_config.reward_token_address), - ], + "solver": self.solvers, + "solver_name": ["S_1", "S_2", "S_3", "S_4"], } ) - self.assertIsNone(pandas.testing.assert_frame_equal(expected, result)) + self.assertIsNone( + pandas.testing.assert_frame_equal( + expected, result.reindex(sorted(result.columns), axis=1) + ) + ) def test_prepare_transfers(self): # Need Example of every possible scenario