Ver 1.0.6 - Support for STEEM/SBD + Steem forks

Full change list:

 - Created Steem coin handler, enabled it by default
     - SteemManager supports sending, balance checking (plus memo balances), health checks,
       customizable RPC nodes per coin (possibly works with Steem forks), and account validation
     - SteemLoader loads transfer history for a Coin's Steem account, properly parses STEEM/SBD
       amounts with no floating point problems, filters out transactions from ourselves etc.
     - Dynamic coin support, using new coin type "Steem Network" (steembase), which may allow
       certain Steem forks such as GOLOS, Whaleshares to work out of the box.

 - Reliability fixes for transaction scanning
     - Memo's can now be up to 1000 chars (prev. 255) in the DB
     - Address's can now be up to 255 chars (prev. 100) in the DB
     - Each Deposit is now inserted with a sub-transaction, fixing a potential issue where one bad
       transaction would cause all added Deposits to be rolled back and rejected.
     - `commands.load_txs` now uses Loader's default `tx_count` instead of statically set to 1000

 - All handler classes now set Decimal to use `ROUND_DOWN` for safety.
 - Fixed Coin config table for Bitcoin handler in documentation
 - Added Steem coin handler to docs, which explains configuration options, with example for the
   custom JSON field settings, and various usage info.
 - Various other small fixes and improvements

docs/source/payments/payments.coin_handlers.Steem.rst

@@ -0,0 +1,34 @@
+.. _Steem Handler:
+
+Steem Coin Handler
+===========================================
+
+Module contents
+---------------
+
+.. automodule:: payments.coin_handlers.Steem
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Submodules
+----------
+
+SteemLoader module
+------------------------------------------------------------
+
+.. automodule:: payments.coin_handlers.Steem.SteemLoader
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+SteemManager module
+-------------------------------------------------------------
+
+.. automodule:: payments.coin_handlers.Steem.SteemManager
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+

docs/source/payments/payments.coin_handlers.rst

@@ -10,6 +10,7 @@ Subpackages
 
     payments.coin_handlers.Bitcoin
     payments.coin_handlers.SteemEngine
+    payments.coin_handlers.Steem
     payments.coin_handlers.base
 
 Module contents

payments/coin_handlers/Bitcoin/BitcoinLoader.py

@@ -17,7 +17,7 @@
 import logging
 import pytz
 from datetime import datetime
-from decimal import Decimal
+from decimal import Decimal, getcontext, ROUND_DOWN
 from typing import Generator, Iterable, List, Dict
 from requests.exceptions import ConnectionError
 from django.utils import timezone
@@ -121,6 +121,7 @@ def clean_txs(self, symbol: str, transactions: Iterable[dict], account: str = No
         """
 
         log.debug('Filtering transactions for %s', symbol)
+
         for tx in transactions:
             t = self._clean_tx(tx, symbol, account)
             if t is None:

payments/coin_handlers/Bitcoin/__init__.py

@@ -11,12 +11,17 @@
 
     For each coin you intend to use with this handler, you should configure it as such:
 
-    coin_type     | This should be set to ``Bitcoind RPC compatible crypto`` (db value: bitcoind)
-    setting_host  | The IP or hostname for the daemon. If not specified, defaults to 127.0.0.1 / localhost
-    setting_port  | The RPC port for the daemon. If not specified, defaults to 8332
-    setting_user  | The rpcuser for the daemon. Generally MUST be specified.
-    setting_pass  | The rpcpassword for the daemon. Generally MUST be specified
-    setting_json  | A JSON string for optional extra config (see below)
+    =============   ==================================================================================================
+    Coin Key        Description
+    =============   ==================================================================================================
+    coin_type       This should be set to ``Bitcoind RPC compatible crypto`` (db value: bitcoind)
+    setting_host    The IP or hostname for the daemon. If not specified, defaults to 127.0.0.1 / localhost
+    setting_port    The RPC port for the daemon. If not specified, defaults to 8332
+    setting_user    The rpcuser for the daemon. Generally MUST be specified.
+    setting_pass    The rpcpassword for the daemon. Generally MUST be specified
+    setting_json    A JSON string for optional extra config (see below)
+    =============   ==================================================================================================
+
 
     Extra JSON (Handler Custom) config options:
 

payments/coin_handlers/Steem/SteemLoader.py

@@ -0,0 +1,192 @@
+"""
+**Copyright**::
+
+    +===================================================+
+    |                 © 2019 Privex Inc.                |
+    |               https://www.privex.io               |
+    +===================================================+
+    |                                                   |
+    |        CryptoToken Converter                      |
+    |                                                   |
+    |        Core Developer(s):                         |
+    |                                                   |
+    |          (+)  Chris (@someguy123) [Privex]        |
+    |                                                   |
+    +===================================================+
+
+"""
+import logging
+from decimal import Decimal, getcontext, ROUND_DOWN
+from typing import Dict, List, Iterable, Generator, Union
+
+import pytz
+from beem.account import Account
+from beem.asset import Asset
+from beem.steem import Steem
+from beem.instance import shared_steem_instance
+from dateutil.parser import parse
+from django.utils import timezone
+
+from payments.coin_handlers import BaseLoader
+from steemengine.helpers import empty
+
+log = logging.getLogger(__name__)
+getcontext().rounding = ROUND_DOWN
+
+
+class SteemLoader(BaseLoader):
+    """
+    SteemLoader - Loads transactions from the Steem network
+
+    Designed for the Steem Network with SBD and STEEM support. May or may not work with other Graphene coins.
+
+    **Copyright**::
+
+      +===================================================+
+      |                 © 2019 Privex Inc.                |
+      |               https://www.privex.io               |
+      +===================================================+
+      |                                                   |
+      |        CryptoToken Converter                      |
+      |                                                   |
+      |        Core Developer(s):                         |
+      |                                                   |
+      |          (+)  Chris (@someguy123) [Privex]        |
+      |                                                   |
+      +===================================================+
+
+    For **additional settings**, please see the module docstring in :py:mod:`coin_handlers.Steem`
+
+    """
+
+    provides = ["STEEM", "SBD"]  # type: List[str]
+    """
+    This attribute is automatically generated by scanning for :class:`models.Coin` s with the type ``steembase``. 
+    This saves us from hard coding specific coin symbols. See __init__.py for populating code.
+    """
+
+    def __init__(self, symbols):
+        super(SteemLoader, self).__init__(symbols=symbols)
+        self.tx_count = 10000
+        self.loaded = False
+        self.rpc = shared_steem_instance()
+
+    @property
+    def settings(self) -> Dict[str, dict]:
+        """To ensure we always get fresh settings from the DB after a reload"""
+        return dict(((sym, c.settings) for sym, c in self.coins.items()))
+
+    def load(self, tx_count=10000):
+        # Unlike other coins, it's important to load a lot of TXs, because many won't actually be transfers
+        # Thus the default TX count for Steem is 10,000
+        self.tx_count = tx_count
+        for symbol, coin in self.coins.items():
+            if not empty(coin.our_account):
+                continue
+            log.warning('The coin %s does not have `our_account` set. Refusing to load transactions.', coin)
+            del self.coins[symbol]
+            self.symbols = [s for s in self.symbols if s != symbol]
+
+    def get_rpc(self, symbol):
+        """
+        Returns a Steem instance for querying data and sending TXs. By default, uses the Beem shared_steem_instance.
+
+        If a custom RPC list is specified in the Coin "custom json" settings, a new instance will be returned with the
+        RPCs specified in the json.
+
+        :param symbol: Coin symbol to get Beem RPC instance for
+        :return beem.steem.Steem: An instance of :class:`beem.steem.Steem` for querying
+        """
+        rpc_list = self.settings[symbol]['json'].get('rpcs')
+        return self.rpc if empty(rpc_list, itr=True) else Steem(node=rpc_list)
+
+    def list_txs(self, batch=0) -> Generator[dict, None, None]:
+        if not self.loaded:
+            self.load()
+        for symbol, c in self.coins.items():
+            acc_name = self.coins[symbol].our_account
+            acc = Account(acc_name, steem_instance=self.get_rpc(symbol))
+            # get_account_history returns a generator with automatic batching, so we don't have to worry about batches.
+            txs = acc.get_account_history(-1, self.tx_count, only_ops=['transfer'])
+            yield from self.clean_txs(symbol=symbol, transactions=txs, account=acc_name)
+
+    def clean_txs(self, symbol: str, transactions: Iterable[dict], account: str = None) -> Generator[dict, None, None]:
+        """
+        Filters a list of transactions `transactions` as required, yields dict's conforming with :class:`models.Deposit`
+
+        - Filters out transactions that are not marked as 'receive'
+        - Filters out mining transactions
+        - Filters by address if `account` is specified
+        - Filters out transactions that don't have enough confirms, and are not reported as 'trusted'
+
+        :param symbol:           Symbol of coin being cleaned
+        :param transactions:     A ``list<dict>`` or generator producing dict's
+        :param account:          If not None, only return TXs sent to this address.
+        :return Generator<dict>: A generator outputting dictionaries formatted as below
+
+        Output Format::
+
+          {
+            txid:str, coin:str (symbol), vout:int,
+            tx_timestamp:datetime, address:str, amount:Decimal
+          }
+
+        """
+
+        log.debug('Filtering transactions for %s', symbol)
+        for tx in transactions:
+            try:
+                t = self.clean_tx(tx, symbol, account)
+                if t is None:
+                    continue
+                yield t
+            except (AttributeError, KeyError) as e:
+                log.warning('Steem TX missing important key? %s', str(e))
+            except:
+                log.exception('Error filtering Steem TX, skipping... TX data: %s', tx)
+
+    @staticmethod
+    def clean_tx(tx: dict, symbol: str, account: str, memo: str = None, memo_case: bool = False) -> Union[dict, None]:
+        """Filters an individual transaction. See :meth:`.clean_txs` for info"""
+        # log.debug(tx)
+        if tx.get('type', 'NOT SET') != 'transfer':
+            log.debug('Steem TX is not transfer. Type is: %s', tx.get('type', 'NOT SET'))
+            return None
+
+        txid = tx.get('trx_id', None)
+
+        _am = tx['amount']  # Transfer ops contain a dict 'amount', containing amount:int, nai:str, precision:int
+
+        amt_sym = str(Asset(_am['nai']).symbol).upper()  # Conv asset ID (e.g. @@000000021) to symbol, i.e. "STEEM"
+
+        if amt_sym != symbol:  # If the symbol doesn't match the symbol we were passed, skip this TX
+            return None
+
+        # Convert integer amount/precision to Decimal's, preventing floating point issues
+        amt_int = Decimal(_am['amount'])
+        amt_prec = Decimal(_am['precision'])
+
+        amt = amt_int / (Decimal(10) ** amt_prec)   # Use precision value to convert from integer amt to decimal amt
+
+        tx_memo = tx.get('memo')
+
+        log.debug('Filtering/cleaning steem transaction, Amt: %f, TXID: %s', amt, txid)
+
+        if tx['to'] != account or tx['from'] == account:
+            return None    # If the transaction isn't to us (account), or it's from ourselves, ignore it.
+        if not empty(memo) and (tx_memo != memo or (not memo_case and tx_memo.lower() != memo.lower())):
+            return None
+
+        d = parse(tx['timestamp'])
+        d = timezone.make_aware(d, pytz.UTC)
+
+        return dict(
+            txid=txid,
+            coin=symbol,
+            vout=int(tx.get('op_in_trx', 0)),
+            tx_timestamp=d,
+            from_account=tx.get('from', None),
+            to_account=tx.get('to', None),
+            memo=tx_memo,
+            amount=Decimal(amt)
+        )