More unit tests, introduce Pytest (#319)

* delphinet uses smaller BLOCKS_PER_ROLL_SNAPSHOT
* added complete unit test for calculating rewards
* Moving unit tests to separate folder
* re-mock requests.get
* replace RpcProviderApiException with ApiProviderException
* more try/catch; migrating to child logging
* add PRPC tests
* update to delphinet, remove old networks
* switch to pytest
* Contributor: utdrmac, Effort=16h
* Reviewer: amzid, Effort=1h

.travis.yml

@@ -28,10 +28,10 @@ addons:
             - python3-sphinx
 
 # command to run tests
-before_script: export PYTHONPATH=$PYTHONPATH:$(pwd)
+before_script: export PYTHONPATH=$PYTHONPATH:$(pwd)/src
 script: 
-    - flake8 src/
-    - python3 -m unittest discover -s src --verbose
+    - flake8 src/ tests/
+    - python3 -m pytest -s tests/ --verbose
     #- make spelling
     #- make linkcheck
     - cd docs

docs/configuration.rst

@@ -135,19 +135,18 @@ Available configuration parameters are:
 
 **rules_map**
   The rules_map is needed to redirect payments. A pre-defined source (left side) is 
-  mindelegation. Pre-defined destinations (right side) are TOF = to founders balance, 
-  TOB = to bakers balance and TOE = to everyone. Variable sources and destinations are 
+  mindelegation. Pre-defined destinations (right side) are: TOF = to founders balance, 
+  TOB = to bakers balance, and TOE = to everyone. Variable sources and destinations are 
   PKHs. New since v8.0 PKH: Dexter enables payouts to Dexter liquidity pools.
-
-  Example:  
-  rules_map:  
-  PKH: TOF (redirects payment from PKH to TOF)
-
-  PHK: TOB (payment will be kept in the baking_address)
-
-  PKH: PKH (redirects payment from PKH to PKH)
-
-  mindelegation: TOE (mindelegation will be shared with everyone)
+
+  Example::
+
+     rules_map:
+       tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ: TOF                                         (redirects payment from tz1T5woJN3r7SV5v2HGDyA5kurhbD9Y8ZKHZ to founders)
+       tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace: TOB                                         (payment to tz1YTMY7Zewx6AMM2h9eCwc8TyXJ5wgn9ace will remain in the bakers balance)
+       tz1V9SpwXaGFiYdDfGJtWjA61EumAH3DwSyT: tz1fgX6oRWQb4HYHUT6eRjW8diNFrqjEfgq7        (redirects payment from tz1V9S... to tz1fgX...)
+       tz1RMmSzPSWPSSaKU193Voh4PosWSZx1C7Hs: Dexter                                      (indicates address is a dexter pool; TRD will send rewards to pool members)
+       mindelegation: TOE                                                                (mindelegation will be shared with everyone)
   
 **plugins**
   This section of the configuration file, along with 'enabled' noted in the example below,

src/Constants.py

@@ -3,8 +3,8 @@
 EXIT_PAYMENT_TYPE = "exit"
 
 PUBLIC_NODE_URL = {"MAINNET": ["https://mainnet-tezos.giganode.io", "https://teznode.letzbake.com"],
-                   "ALPHANET": ["https://tezos-dev.cryptonomic-infra.tech", "https://testnet-tezos.giganode.io"],
-                   "ZERONET": ["https://rpczero.tzbeta.net"]}
+                   "DELPHINET": ["https://tezos-dev.cryptonomic-infra.tech", "https://delphinet-tezos.giganode.io"]
+                   }
 
 TEZOS_RPC_PORT = 8732
 

src/NetworkConfiguration.py

@@ -7,18 +7,16 @@
 default_network_config_map = {
     'MAINNET': {'NAME': 'MAINNET', 'NB_FREEZE_CYCLE': 5, 'BLOCK_TIME_IN_SEC': 60, 'BLOCKS_PER_CYCLE': 4096,
                 'BLOCKS_PER_ROLL_SNAPSHOT': 256},
-    'ALPHANET': {'NAME': 'ALPHANET', 'NB_FREEZE_CYCLE': 3, 'BLOCK_TIME_IN_SEC': 30, 'BLOCKS_PER_CYCLE': 2048,
-                 'BLOCKS_PER_ROLL_SNAPSHOT': 256},
-    'ZERONET': {'NAME': 'ZERONET', 'NB_FREEZE_CYCLE': 5, 'BLOCK_TIME_IN_SEC': 20, 'BLOCKS_PER_CYCLE': 128,
-                'BLOCKS_PER_ROLL_SNAPSHOT': 8},
+    'DELPHINET': {'NAME': 'DELPHINET', 'NB_FREEZE_CYCLE': 3, 'BLOCK_TIME_IN_SEC': 30, 'BLOCKS_PER_CYCLE': 2048,
+                  'BLOCKS_PER_ROLL_SNAPSHOT': 128},
 }
 
 CONSTANTS_PATH = "/chains/main/blocks/head/context/constants"
 CONSTANTS_RPC = "rpc get " + CONSTANTS_PATH
 
 PUBLIC_NODE_BASE = "https://{}-tezos.giganode.io"
 PUBLIC_NODE_RPC = PUBLIC_NODE_BASE + CONSTANTS_PATH
-PUBLIC_NODE_PREFIX = {"MAINNET": "mainnet", "ALPHANET": "testnet", "ZERONET": "labnet"}
+PUBLIC_NODE_PREFIX = {"MAINNET": "mainnet", "DELPHINET": "delphinet"}
 
 
 def init_network_config(network_name, config_client_manager, node_addr):
@@ -46,10 +44,6 @@ def init_network_config(network_name, config_client_manager, node_addr):
     return default_network_config_map
 
 
-def is_mainnet(nw_name):
-    return nw_name == 'MAINNET'
-
-
 def get_network_config_from_local_node(config_client_manager, node_addr):
     _, response_constants = config_client_manager.send_request(CONSTANTS_RPC)
     constants = parse_json_response(response_constants)

src/api/provider_factory.py

@@ -36,4 +36,5 @@ def newBlockApi(self, network_config, node_url, api_base_url=None):
             return TzStatsBlockApiImpl(network_config)
         elif self.provider == 'tzkt':
             return TzKTBlockApiImpl(network_config, base_url=api_base_url)
+
         raise Exception("No supported reward data provider : {}".format(self.provider))

src/calc/calculate_phaseMerge.py

@@ -20,6 +20,7 @@ def calculate(self, reward_logs):
 
         address_set = set(rl.paymentaddress for rl in self.filterskipped(reward_logs))
         payment_address_list_dict = {addr: [] for addr in address_set}
+
         # group payments by paymentaddress
         for rl in self.filterskipped(reward_logs):
             payment_address_list_dict[rl.paymentaddress].append(rl)

src/calc/phased_payment_calculator.py

@@ -7,7 +7,9 @@
 from calc.calculate_phase4 import CalculatePhase4
 from calc.calculate_phase_final import CalculatePhaseFinal
 from model.reward_log import TYPE_FOUNDERS_PARENT, TYPE_OWNERS_PARENT, cmp_by_type_balance
-from pay.payment_consumer import logger
+from log_config import main_logger
+
+logger = main_logger.getChild("phased_calculator")
 
 MINOR_DIFF = 10
 MINOR_RATIO_DIFF = 1e-6
@@ -41,6 +43,7 @@ def __init__(self, founders_map, owners_map, service_fee_calculator, min_delegat
     # founders reward = delegators fee = total reward - delegators reward
     ####
     def calculate(self, reward_provider_model):
+
         phase0 = CalculatePhase0(reward_provider_model)
         rwrd_logs, total_rwrd_amnt = phase0.calculate()
 

src/configure.py

@@ -33,13 +33,13 @@
 logger = main_logger
 
 messages = {
-    'hello': 'This application will help you configure TRD payouts for your bakery. Type enter to continue',
+    'hello': 'This application will help you configure TRD to manage payouts for your bakery. Type enter to continue',
     'bakingaddress': 'Specify your baking address public key hash (Processing may take a few seconds)',
     'paymentaddress': 'Specify your payment PKH/alias. Available aliases:{}',
     'servicefee': 'Specify bakery fee [0:100]',
     'foundersmap': "Specify FOUNDERS in form 'PKH1':share1,'PKH2':share2,... (Mind quotes) Type enter to leave empty",
     'ownersmap': "Specify OWNERS in form 'pk1':share1,'pkh2':share2,... (Mind quotes) Type enter to leave empty",
-    'mindelegation': "Specify minimum delegation amount in tezos. Type enter for 0",
+    'mindelegation': "Specify minimum delegation amount in tez. Type enter for 0",
     'mindelegationtarget': "Specify where the reward for delegators failing to satisfy minimum delegation amount go. TOB: leave at balance, TOF: to founders, TOE: to everybody, default is TOB",
     'exclude': "Add excluded address in form of PKH,target. Share of the exluded address will go to target. Possbile targets are= TOB: leave at balance, TOF: to founders, TOE: to everybody. Type enter to skip",
     'redirect': "Add redirected address in form of PKH1,PKH2. Payments for PKH1 will go to PKH2. Type enter to skip",
@@ -389,7 +389,7 @@ def main(args):
     client_path = get_client_path([x.strip() for x in args.executable_dirs.split(',')],
                                   args.docker, args.network)
 
-    logger.debug("Tezos client path is {}".format(client_path))
+    logger.debug("tezos-client path is {}".format(client_path))
 
     # 4. get network config
     config_client_manager = SimpleClientManager(client_path, args.node_addr)

src/exception/api_provider.py

@@ -1,2 +1,7 @@
+# API providers should throw this exception when an
+# error condition happens while fetching data from source
+#
+# API providers may subclass for their specific needs
+#
 class ApiProviderException(Exception):
     pass

src/launch_common.py

@@ -110,8 +110,8 @@ def add_argument_payment_offset(parser):
 
 def add_argument_network(parser):
     parser.add_argument("-N", "--network",
-                        help="Network name. Default is Mainnet. The test network of tezos is referred to as Alphanet even if the name changes with each protocol upgrade.",
-                        choices=['MAINNET', 'ZERONET', 'ALPHANET'],
+                        help="Network name. Default is Mainnet. The current test network of tezos is DELPHINET.",
+                        choices=['MAINNET', 'DELPHINET'],
                         default='MAINNET')
 
 
@@ -140,7 +140,7 @@ def add_argument_reports_base(parser):
 
 
 def add_argument_config_dir(parser):
-    parser.add_argument("-f", "--config_dir", help="Directory to find baking configurations", default='~/pymnt/cfg')
+    parser.add_argument("-f", "--config_dir", help="Directory to find baking configuration", default='~/pymnt/cfg')
 
 
 def add_argument_dry(parser):

src/model/reward_log.py

@@ -61,9 +61,9 @@ def skip(self, desc, phase):
         return self
 
     def __repr__(self) -> str:
-        return "Address: {} ({}), Type: {}, SB: {}, CB: {}, Skipped: {}, NA: {}".format(
+        return "Address: {} ({}), T: {}, SB: {}, CB: {}, Amt: {}, Skp: {}, NA: {}".format(
             self.address, self.paymentaddress, self.type,
-            self.staking_balance, self.current_balance,
+            self.staking_balance, self.current_balance, self.amount,
             self.skipped, self.needs_activation)
 
     @staticmethod

src/pay/double_payment_check.py

@@ -15,6 +15,7 @@ def check_past_payment(payments_root, payment_cycle):
 
     # new payments are reported to csv files
     payment_file = payment_report_file_path(payments_root, payment_cycle, 0)
+
     if os.path.isfile(payment_file):
         return "Payment report for cycle {} is present. No payment will be run for the cycle. Check '{}'" \
             .format(payment_cycle, payment_file)

src/pay/payment_consumer.py

@@ -14,7 +14,7 @@
 from util.csv_payment_file_parser import CsvPaymentFileParser
 from util.dir_utils import payment_report_file_path, get_busy_file
 
-logger = main_logger
+logger = main_logger.getChild("payment_consumer")
 MUTEZ = 1e6
 
 

src/pay/payment_producer.py

@@ -11,7 +11,6 @@
 from model.reward_log import RewardLog
 from model.rules_model import RulesModel
 from exception.api_provider import ApiProviderException
-from rpc.rpc_reward_api import RpcRewardApiError
 from requests import ReadTimeout, ConnectTimeout
 from pay.double_payment_check import check_past_payment
 from pay.payment_batch import PaymentBatch
@@ -21,7 +20,7 @@
 from util.dir_utils import get_calculation_report_file, get_failed_payments_dir, PAYMENT_FAILED_DIR, PAYMENT_DONE_DIR, \
     remove_busy_file, BUSY_FILE
 
-logger = main_logger
+logger = main_logger.getChild("payment_producer")
 
 MUTEZ = 1e+6
 BOOTSTRAP_SLEEP = 8
@@ -123,16 +122,22 @@ def run(self):
         if self.run_mode == RunMode.FOREVER:
             self.retry_fail_thread.start()
 
-        current_cycle = self.block_api.get_current_cycle()
-        pymnt_cycle = self.initial_payment_cycle
+        try:
+            current_cycle = self.block_api.get_current_cycle()
+            pymnt_cycle = self.initial_payment_cycle
+        except ApiProviderException as a:
+            logger.error("Unable to fetch current cycle, {:s}. Exiting.".format(str(a)))
+            self.exit()
+            return
 
         # if non-positive initial_payment_cycle, set initial_payment_cycle to
         # 'current cycle - abs(initial_cycle) - (NB_FREEZE_CYCLE+1)'
         if self.initial_payment_cycle <= 0:
             pymnt_cycle = current_cycle - abs(self.initial_payment_cycle) - (self.nw_config['NB_FREEZE_CYCLE'] + 1)
             logger.debug("Payment cycle is set to {}".format(pymnt_cycle))
 
-        get_verbose_log_helper().reset(pymnt_cycle)
+        if get_verbose_log_helper() is not None:
+            get_verbose_log_helper().reset(pymnt_cycle)
 
         while not self.exiting and self.life_cycle.is_running():
 
@@ -157,7 +162,7 @@ def run(self):
                     os.makedirs(self.calculations_dir)
 
                 logger.debug("Checking for pending payments : payment_cycle <= current_cycle - (self.nw_config['NB_FREEZE_CYCLE'] + 1) - self.release_override")
-                logger.info("Checking for pending payments : checking {} <= {} - ({} + 1) - {}". format(pymnt_cycle, current_cycle, self.nw_config['NB_FREEZE_CYCLE'], self.release_override))
+                logger.info("Checking for pending payments : checking {} <= {} - ({} + 1) - {}".format(pymnt_cycle, current_cycle, self.nw_config['NB_FREEZE_CYCLE'], self.release_override))
 
                 # payments should not pass beyond last released reward cycle
                 if pymnt_cycle <= current_cycle - (self.nw_config['NB_FREEZE_CYCLE'] + 1) - self.release_override:
@@ -229,10 +234,10 @@ def run(self):
                     # wait until current cycle ends
                     self.wait_for_blocks(nb_blocks_remaining)
 
-            except (ApiProviderException, RpcRewardApiError, ReadTimeout, ConnectTimeout) as e:
-                logger.debug("{:s} error at payment producer loop".format(self.reward_api.name), exc_info=True)
-                logger.warning("{:s} error at payment producer loop: '{:s}', will try again.".format(
-                               self.reward_api.name, str(e)))
+            except (ApiProviderException, ReadTimeout, ConnectTimeout) as e:
+                logger.debug("{:s} error at payment producer loop: '{:s}'".format(self.reward_api.name, str(e)), exc_info=True)
+                logger.error("{:s} error at payment producer loop: '{:s}', will try again.".format(
+                             self.reward_api.name, str(e)))
 
             except Exception as e:
                 logger.debug("Unknown error in payment producer loop: {:s}".format(str(e)), exc_info=True)
@@ -266,7 +271,7 @@ def try_to_pay(self, pymnt_cycle, expected_reward=False):
             # 2- calculate rewards
             reward_logs, total_amount = self.payment_calc.calculate(reward_model)
 
-            # set cycle info
+            # 3- set cycle info
             for rl in reward_logs:
                 rl.cycle = pymnt_cycle
             total_amount_to_pay = sum([rl.amount for rl in reward_logs if rl.payable])
@@ -294,11 +299,16 @@ def try_to_pay(self, pymnt_cycle, expected_reward=False):
             elif total_amount_to_pay == 0:
                 logger.info("Total payment amount is 0. Nothing to pay!")
 
-            return True
-
+        except ApiProviderException as a:
+            logger.error("[try_to_pay] API provider error {:s}".format(str(a)))
+            raise a from a
         except Exception as e:
+            logger.error("[try_to_pay] Generic exception {:s}".format(str(e)))
             raise e from e
 
+        # Either succeeded or raised exception
+        return True
+
     def wait_for_blocks(self, nb_blocks_remaining):
         for x in range(nb_blocks_remaining):
             sleep(self.nw_config['BLOCK_TIME_IN_SEC'])

src/rpc/rpc_block_api.py

@@ -1,8 +1,9 @@
 import requests
 from api.block_api import BlockApi
+from exception.api_provider import ApiProviderException
 from log_config import main_logger
 
-logger = main_logger
+logger = main_logger.getChild("rpc_block_api")
 
 COMM_HEAD = "{}/chains/main/blocks/head"
 COMM_REVELATION = "{}/chains/main/blocks/head/context/contracts/{}/manager_key"
@@ -15,18 +16,28 @@ def __init__(self, nw, node_url):
         self.node_url = node_url
         logger.debug("RpcBlockApiImpl - node_url {}".format(self.node_url))
 
-    def get_current_level(self):
-        response = requests.get(COMM_HEAD.format(self.node_url), timeout=5)
-        head = response.json()
-        current_level = int(head["metadata"]["level"]["level"])
-        return current_level
-
-    def get_revelation(self, pkh):
-        response = requests.get(COMM_REVELATION.format(self.node_url, pkh), timeout=5)
-        manager_key = response.json()
-        logger.debug("Manager key is '{}'".format(manager_key))
-        bool_revelation = manager_key and manager_key != 'null'
-        return bool_revelation
+    def get_current_level(self, verbose=False):
+        try:
+            response = requests.get(COMM_HEAD.format(self.node_url), timeout=5)
+            head = response.json()
+            current_level = int(head["metadata"]["level"]["level"])
+            return current_level
+        except requests.exceptions.RequestException as e:
+            message = "[RpcBlockApiImpl] - Unable to fetch /head: {:s}".format(str(e))
+            logger.error(message)
+            raise ApiProviderException(message)
+
+    def get_revelation(self, pkh, verbose=False):
+        try:
+            response = requests.get(COMM_REVELATION.format(self.node_url, pkh), timeout=5)
+            manager_key = response.json()
+            logger.debug("Manager key is '{}'".format(manager_key))
+            bool_revelation = manager_key and manager_key != 'null'
+            return bool_revelation
+        except requests.exceptions.RequestException as e:
+            message = "[RpcBlockApiImpl] - Unable to fetch revelation: {:s}".format(str(e))
+            logger.error(message)
+            raise ApiProviderException(message)
 
 
 def test_get_revelation():