Ithaca increase test coverage and fix tzstats (#610)

* Reactivate tests after ithaca merge
* Fix/Update rpc reward test
* Cover rpc reward api with tests
* Fixed test_expected_rewards
* Update public rpc node
* Fix integration tests
* Check if delegates amount is sorted and correct
* Update tzstats api consistency check
* Update regression tests
* Fix rpc tests
* Fix args validator
* Update api consistency check
* Fix mock reward api tests
* Contributor: vkresch, Effort=30h
* Reviewer: rvermootenct/jdsika, Effort=3h/1h

.gitignore

@@ -15,6 +15,7 @@ fee.ini*
 .pytest_cache
 .python-version
 lock
+.coverage
 
 # State diagram outputs for sphinx documentation
 config_cycle_state_diagram.png

src/Constants.py

@@ -39,10 +39,10 @@
 PRIVATE_SIGNER_URL = "http://{}:{}".format(LOCAL_HOST, SIGNER_PORT)
 PRIVATE_NODE_URL = "http://{}:{}".format(LOCAL_HOST, TEZOS_RPC_PORT)
 
-# Public RPC
+# Public RPC https://midl-dev.medium.com/alternatives-to-tezos-giganode-bb67b43945ba
 PUBLIC_NODE_URL = {
-    "MAINNET": "https://rpc.tzkt.io/mainnet",
-    CURRENT_TESTNET: "https://rpc.tzkt.io/{}".format(CURRENT_TESTNET.lower()),
+    "MAINNET": "https://mainnet.smartpy.io",
+    CURRENT_TESTNET: "https://testnet.smartpy.io",
 }
 
 # TzStats

src/calc/phased_payment_calculator.py

@@ -159,7 +159,7 @@ def calculate(self, reward_provider_model, adjustments=None, rerun=False):
             return rwrd_logs, total_rwrd_amnt
 
         # sort rewards according to type and balance
-        rwrd_logs.sort(key=functools.cmp_to_key(cmp_by_type_balance))
+        rwrd_logs.sort(key=lambda rl: (rl.type, -rl.staking_balance))
 
         # check if there is difference between sum of calculated amounts and total_rewards
         total_delegator_amounts = int(

src/pay/payment_consumer.py

@@ -146,7 +146,7 @@ def _consume_batch(self, payment_batch):
                 payment_items, self.reactivate_zeroed
             )
 
-            payment_items.sort(key=functools.cmp_to_key(cmp_by_type_balance))
+            payment_items.sort(key=lambda rl: (rl.type, -rl.staking_balance))
 
             batch_payer = BatchPayer(
                 self.node_addr,
@@ -357,6 +357,7 @@ def add_transaction_fees_to_calculation_report(self, payment_logs, payment_cycle
             logger.info("Simulated transaction_fees added to calculations file.")
         else:
             logger.info("Calculations file not modified.")
+            return
 
         return report_file
 

src/tzkt/tzkt_api.py

@@ -100,6 +100,12 @@ def _request(self, path, **params):
 
         return res
 
+    def get_current_cycle(self):
+        return self.get_head()["cycle"]
+
+    def get_current_level(self):
+        return self.get_head()["level"]
+
     def get_head(self) -> dict:
         """
         Returns indexer head and synchronization status.

src/tzkt/tzkt_reward_api.py

@@ -32,9 +32,15 @@ def get_rewards_for_cycle_map(self, cycle, rewards_type) -> RewardProviderModel:
         # calculate estimated rewards
         num_blocks = split["blocks"] + split["missedBlocks"] + split["futureBlocks"]
 
-        # warning: this value will be 0 after the cycle ran.
+        # warning: futureEndorsementRewards will be 0 after the cycle ran.
+        # if we query a past cycle then we set the potential_endorsement_rewards with the
+        # actual endorsement reward for consistency with other reward apis
         # But after cycle ran, we never pay estimates, so this value will not be used.
-        potential_endorsement_rewards = split["futureEndorsementRewards"]
+        potential_endorsement_rewards = (
+            split["futureEndorsementRewards"]
+            if self.api.get_current_cycle() <= cycle
+            else split["endorsementRewards"]
+        )
 
         # rewards earned (excluding equivocation losses)
         rewards_and_fees = (

src/tzstats/tzstats_api_constants.py

@@ -1,28 +1,71 @@
+# https://tzstats.com/docs/api#income-table
+# 0:    64698,            // row_id
+# 1:    150,              // cycle
+# 2:    278469,           // account_id
+# 3:    3576,             // rolls
+# 4:    3183854.077395,   // balance
+# 5:    25426700.898205,  // delegated
+# 6:    0,                // active_stake
+# 7:    4,                // n_delegations
+# 8:    199,              // n_baking_rights
+# 9:    6507,             // n_endorsing_rights
+# 10:    -292.000000,      // luck
+# 11:    98.230000,        // luck_percent
+# 12:    99.87,            // performance_percent
+# 13:    100.00,           // contribution_percent
+# 14:    201,              // n_blocks_baked
+# 15:    201,              // n_blocks_proposed
+# 16:    0,                // n_blocks_not_baked
+# 17:    3293,             // n_blocks_endorsed
+# 18:    0,                // n_blocks_not_endorsed
+# 19:    6507,             // n_slots_endorsed
+# 20:    15,               // n_seeds_revealed
+# 21:    16198.000000,     // expected_income
+# 22:    16176.875000,     // total_income
+# 23:    519360.000000,    // total_deposits
+# 24:    3216.000000,      // baking_income
+# 25:    12959.000000,     // endorsing_income
+# 26:    0.000000,         // accusation_income
+# 27:    1.875000,         // seed_income
+# 28:    1.680381,         // fees_income
+# 29:    0,                // total_loss
+# 30:    0,                // accusation_loss
+# 31:    0,                // seed_loss
+# 32:    0,                // endorsing_loss
+# 33:    0,                // lost_accusation_fees
+# 34:    0,                // lost_accusation_rewards
+# 35:    0,                // lost_accusation_deposits
+# 36:    0,                // lost_seed_fees
+# 37:    0,                // lost_seed_rewards
+# 38:    "tz2TSvNTh2epDMhZHrw73nV9piBX7kLZ9K9m",  // address
+# 39:    1568887343000,    // start_time
+# 40:    1569136305000     // end_time
+
 # Income/Rewards Breakdown
-idx_income_expected_income = 17
-idx_income_total_income = 18
-idx_income_total_bonds = 19
+idx_income_expected_income = 21
+idx_income_total_income = 22
 
-idx_income_baking_income = 21
-idx_income_endorsing_income = 22
-idx_income_double_baking_income = 23
-idx_income_double_endorsing_income = 22  # missing, replaced by accusation income
-idx_income_seed_income = 24
-idx_income_fees_income = 25
+idx_income_baking_income = 24
+idx_income_endorsing_income = 25
+idx_income_accusation_income = 26
+idx_income_seed_income = 27
+idx_income_fees_income = 28
 idx_income_missed_baking_income = 29  # Missing for now?
 idx_income_missed_endorsing_income = 30  # Missing for now?
 idx_income_stolen_baking_income = 31  # Missing for now?
 
-idx_income_total_lost = 26
-idx_income_lost_accusation_fees = 27  # missing?
-idx_income_lost_accusation_rewards = 34  # missing?
-idx_income_lost_accusation_deposits = 35  # missing?
-idx_income_lost_revelation_fees = 36  # missing?
-idx_income_lost_revelation_rewards = 37  # missing?
+idx_income_total_loss = 29
+idx_income_lost_accusation_fees = 33
+idx_income_lost_accusation_rewards = 34
+idx_income_lost_accusation_deposits = 35
+idx_income_lost_seed_fees = 36
+idx_income_lost_seed_rewards = 37
 
 # Rights
-idx_n_baking_rights = 9
-idx_n_active_stake = 7
+idx_n_baking_rights = 8
+idx_n_blocks_baked = 14
+idx_n_blocks_not_baked = 16
+idx_n_active_stake = 6
 
 # Cycle Snapshot
 idx_balance = 0

src/tzstats/tzstats_block_api.py

@@ -26,7 +26,9 @@ def get_current_cycle_and_level(self):
         verbose_logger.debug("Response from tzstats is: {}".format(root))
 
         current_cycle = int(root["cycle"])
-        current_level = int(root["height"])
+        current_level = (
+            int(root["height"]) + 1
+        )  # Adding one to be in synch with the tzkt API
 
         return (current_cycle, current_level)
 

src/tzstats/tzstats_reward_api.py

@@ -4,6 +4,7 @@
 from log_config import main_logger
 from model.reward_provider_model import RewardProviderModel
 from tzstats.tzstats_reward_provider_helper import TzStatsRewardProviderHelper
+from Constants import MUTEZ_PER_TEZ
 from Dexter import dexter_utils as dxtz
 
 logger = main_logger
@@ -39,13 +40,16 @@ def get_rewards_for_cycle_map(self, cycle, rewards_type):
         number_of_endorsements_per_cycle = (
             self.blocks_per_cycle * self.consensus_committee_size
         )
-        potential_endorsement_rewards = (
+
+        # https://tezos-dev.slack.com/archives/CV5NX7F2L/p1649433246273169?thread_ts=1648854391.875409&cid=CV5NX7F2L
+        potential_endorsement_rewards = int(
             math.floor(
                 delegate_staking_balance
                 * number_of_endorsements_per_cycle
                 / total_active_stake
             )
             * self.endorsing_reward_per_slot
+            / MUTEZ_PER_TEZ
         )
 
         snapshot_level = self.helper.get_snapshot_level(cycle)

src/tzstats/tzstats_reward_provider_helper.py

@@ -10,25 +10,23 @@
     idx_income_total_income,
     idx_income_lost_accusation_fees,
     idx_income_lost_accusation_rewards,
-    idx_income_lost_revelation_fees,
-    idx_income_lost_revelation_rewards,
-    idx_delegator_address,
-    idx_balance,
-    idx_baker_delegated,
     idx_cb_delegator_address,
     idx_cb_current_balance,
+    idx_n_active_stake,
+    idx_income_lost_accusation_deposits,
+    idx_income_lost_seed_fees,
+    idx_income_lost_seed_rewards,
     idx_income_missed_baking_income,
     idx_income_missed_endorsing_income,
-    idx_income_double_baking_income,
-    idx_income_double_endorsing_income,
-    idx_n_active_stake,
 )
 from Constants import TZSTATS_PUBLIC_API_URL, MUTEZ_PER_TEZ
 
 logger = main_logger
 
 rewards_split_call = "/tables/income?address={}&cycle={}"
 
+reward_snapshot = "/explorer/bakers/{}/snapshot/{}"
+
 delegators_call = "/tables/snapshot?cycle={}&is_selected=1&baker={}&columns=balance,delegated,address&limit=50000"
 
 batch_current_balance_call = (
@@ -43,6 +41,8 @@
 
 balance_LP_call = "/explorer/bigmap/{}/values?limit=100&offset={}&block={}"
 
+tip = "/explorer/tip"
+
 
 def split(input, n):
     for i in range(0, len(input), n):
@@ -105,13 +105,14 @@ def get_rewards_for_cycle(self, cycle):
         root["equivocation_losses"] = int(
             MUTEZ_PER_TEZ
             * (
-                0
-                # float(resp[idx_income_lost_accusation_fees])
-                # + float(resp[idx_income_lost_accusation_rewards])
-                # + float(resp[idx_income_lost_revelation_fees])
-                # + float(resp[idx_income_lost_revelation_rewards])
+                float(resp[idx_income_lost_accusation_fees])
+                + float(resp[idx_income_lost_accusation_rewards])
+                + float(resp[idx_income_lost_accusation_deposits])
+                + float(resp[idx_income_lost_seed_fees])
+                + float(resp[idx_income_lost_seed_rewards])
             )
         )
+        # TODO: Find out how to calculate denuciation rewards via tzstats
         root["denunciation_rewards"] = int(
             MUTEZ_PER_TEZ
             * (
@@ -124,18 +125,14 @@ def get_rewards_for_cycle(self, cycle):
         root["offline_losses"] = int(
             MUTEZ_PER_TEZ
             * (
-                0
-                # float(resp[idx_income_missed_baking_income])
-                # + float(resp[idx_income_missed_endorsing_income])
+                float(resp[idx_income_missed_baking_income])
+                + float(resp[idx_income_missed_endorsing_income])
             )
         )
 
-        #
         # Get staking balances of delegators at snapshot block
         #
-        uri = self.api + delegators_call.format(
-            cycle - self.preserved_cycles - 1, self.baking_address
-        )
+        uri = self.api + reward_snapshot.format(self.baking_address, cycle)
         sleep(0.5)  # be nice to tzstats
 
         verbose_logger.debug(
@@ -154,24 +151,12 @@ def get_rewards_for_cycle(self, cycle):
 
         resp = resp.json()
 
-        for delegator in resp:
-
-            if delegator[idx_delegator_address] == self.baking_address:
-                root["delegate_staking_balance"] = int(
-                    MUTEZ_PER_TEZ
-                    * (
-                        float(delegator[idx_balance])
-                        + float(delegator[idx_baker_delegated])
-                    )
-                )
-            else:
-                delegator_info = {"staking_balance": 0, "current_balance": 0}
-                delegator_info["staking_balance"] = int(
-                    MUTEZ_PER_TEZ * float(delegator[idx_balance])
-                )
-                root["delegators_balances"][
-                    delegator[idx_delegator_address]
-                ] = delegator_info
+        root["delegate_staking_balance"] = int(resp["staking_balance"])
+        for delegator in resp["delegators"]:
+            delegator_info = {"staking_balance": 0, "current_balance": 0}
+            delegator_info["staking_balance"] = int(delegator["balance"])
+            if delegator_info["staking_balance"] > 0:
+                root["delegators_balances"][delegator["address"]] = delegator_info
 
         #
         # Get current balance of delegates
@@ -235,7 +220,7 @@ def get_rewards_for_cycle(self, cycle):
         if len(need_curr_balance_fetch) > 0:
             split_addresses = split(need_curr_balance_fetch, 50)
             for list_address in split_addresses:
-                list_curr_balances = self.__fetch_current_balance(list_address)
+                list_curr_balances = self.fetch_current_balance(list_address)
                 for d in list_address:
                     root["delegators_balances"][d][
                         "current_balance"
@@ -261,7 +246,7 @@ def update_current_balances(self, reward_logs):
         split_addresses = split(reward_logs, 50)
         for list_address in split_addresses:
             addresses = [x.address for x in list_address]
-            list_curr_balances = self.__fetch_current_balance(addresses)
+            list_curr_balances = self.fetch_current_balance(addresses)
             for d in list_address:
                 d.current_balance = list_curr_balances[d.address]
 
@@ -288,10 +273,17 @@ def get_cycle_total_stake(self, cycle):
             # This means something went wrong.
             raise ApiProviderException("GET {} {}".format(uri, resp.status_code))
 
-        staking_supply = resp.json()["staking_supply"]
+        resp_json = resp.json()
+        staking_supply = resp_json["snapshot_cycle"]["staking_supply"]
         return staking_supply
 
-    def __fetch_current_balance(self, address_list):
+    def get_current_cycle(self):
+        uri = self.api + tip
+        resp = requests.get(uri, timeout=5)
+        root = resp.json()
+        return root["cycle"]
+
+    def fetch_current_balance(self, address_list):
         param_txt = ""
         for address in address_list:
             param_txt += address + ","
@@ -370,6 +362,6 @@ def get_liquidity_providers_list(self, big_map_id, snapshot_block):
     def update_current_balances_dexter(self, balanceMap):
         split_addresses = split(list(balanceMap.keys()), 50)
         for list_address in split_addresses:
-            list_curr_balances = self.__fetch_current_balance(list_address)
+            list_curr_balances = self.fetch_current_balance(list_address)
             for d in list_address:
                 balanceMap[d].update({"current_balance": list_curr_balances[d]})