diff --git a/docs/source/payments/payments.coin_handlers.Steem.rst b/docs/source/payments/payments.coin_handlers.Steem.rst new file mode 100644 index 0000000..e41550e --- /dev/null +++ b/docs/source/payments/payments.coin_handlers.Steem.rst @@ -0,0 +1,34 @@ +.. _Steem Handler: + +Steem Coin Handler +=========================================== + +Module contents +--------------- + +.. automodule:: payments.coin_handlers.Steem + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +SteemLoader module +------------------------------------------------------------ + +.. automodule:: payments.coin_handlers.Steem.SteemLoader + :members: + :undoc-members: + :show-inheritance: + +SteemManager module +------------------------------------------------------------- + +.. automodule:: payments.coin_handlers.Steem.SteemManager + :members: + :undoc-members: + :show-inheritance: + + + diff --git a/docs/source/payments/payments.coin_handlers.rst b/docs/source/payments/payments.coin_handlers.rst index 7b3f3ca..1d06b66 100644 --- a/docs/source/payments/payments.coin_handlers.rst +++ b/docs/source/payments/payments.coin_handlers.rst @@ -10,6 +10,7 @@ Subpackages payments.coin_handlers.Bitcoin payments.coin_handlers.SteemEngine + payments.coin_handlers.Steem payments.coin_handlers.base Module contents diff --git a/payments/coin_handlers/Bitcoin/BitcoinLoader.py b/payments/coin_handlers/Bitcoin/BitcoinLoader.py index ab30200..e47b87a 100644 --- a/payments/coin_handlers/Bitcoin/BitcoinLoader.py +++ b/payments/coin_handlers/Bitcoin/BitcoinLoader.py @@ -17,7 +17,7 @@ import logging import pytz from datetime import datetime -from decimal import Decimal +from decimal import Decimal, getcontext, ROUND_DOWN from typing import Generator, Iterable, List, Dict from requests.exceptions import ConnectionError from django.utils import timezone @@ -121,6 +121,7 @@ def clean_txs(self, symbol: str, transactions: Iterable[dict], account: str = No """ log.debug('Filtering transactions for %s', symbol) + for tx in transactions: t = self._clean_tx(tx, symbol, account) if t is None: diff --git a/payments/coin_handlers/Bitcoin/__init__.py b/payments/coin_handlers/Bitcoin/__init__.py index 893b68b..e64d2f9 100644 --- a/payments/coin_handlers/Bitcoin/__init__.py +++ b/payments/coin_handlers/Bitcoin/__init__.py @@ -11,12 +11,17 @@ For each coin you intend to use with this handler, you should configure it as such: - coin_type | This should be set to ``Bitcoind RPC compatible crypto`` (db value: bitcoind) - setting_host | The IP or hostname for the daemon. If not specified, defaults to 127.0.0.1 / localhost - setting_port | The RPC port for the daemon. If not specified, defaults to 8332 - setting_user | The rpcuser for the daemon. Generally MUST be specified. - setting_pass | The rpcpassword for the daemon. Generally MUST be specified - setting_json | A JSON string for optional extra config (see below) + ============= ================================================================================================== + Coin Key Description + ============= ================================================================================================== + coin_type This should be set to ``Bitcoind RPC compatible crypto`` (db value: bitcoind) + setting_host The IP or hostname for the daemon. If not specified, defaults to 127.0.0.1 / localhost + setting_port The RPC port for the daemon. If not specified, defaults to 8332 + setting_user The rpcuser for the daemon. Generally MUST be specified. + setting_pass The rpcpassword for the daemon. Generally MUST be specified + setting_json A JSON string for optional extra config (see below) + ============= ================================================================================================== + Extra JSON (Handler Custom) config options: diff --git a/payments/coin_handlers/Steem/SteemLoader.py b/payments/coin_handlers/Steem/SteemLoader.py new file mode 100644 index 0000000..750c3ef --- /dev/null +++ b/payments/coin_handlers/Steem/SteemLoader.py @@ -0,0 +1,192 @@ +""" +**Copyright**:: + + +===================================================+ + | © 2019 Privex Inc. | + | https://www.privex.io | + +===================================================+ + | | + | CryptoToken Converter | + | | + | Core Developer(s): | + | | + | (+) Chris (@someguy123) [Privex] | + | | + +===================================================+ + +""" +import logging +from decimal import Decimal, getcontext, ROUND_DOWN +from typing import Dict, List, Iterable, Generator, Union + +import pytz +from beem.account import Account +from beem.asset import Asset +from beem.steem import Steem +from beem.instance import shared_steem_instance +from dateutil.parser import parse +from django.utils import timezone + +from payments.coin_handlers import BaseLoader +from steemengine.helpers import empty + +log = logging.getLogger(__name__) +getcontext().rounding = ROUND_DOWN + + +class SteemLoader(BaseLoader): + """ + SteemLoader - Loads transactions from the Steem network + + Designed for the Steem Network with SBD and STEEM support. May or may not work with other Graphene coins. + + **Copyright**:: + + +===================================================+ + | © 2019 Privex Inc. | + | https://www.privex.io | + +===================================================+ + | | + | CryptoToken Converter | + | | + | Core Developer(s): | + | | + | (+) Chris (@someguy123) [Privex] | + | | + +===================================================+ + + For **additional settings**, please see the module docstring in :py:mod:`coin_handlers.Steem` + + """ + + provides = ["STEEM", "SBD"] # type: List[str] + """ + This attribute is automatically generated by scanning for :class:`models.Coin` s with the type ``steembase``. + This saves us from hard coding specific coin symbols. See __init__.py for populating code. + """ + + def __init__(self, symbols): + super(SteemLoader, self).__init__(symbols=symbols) + self.tx_count = 10000 + self.loaded = False + self.rpc = shared_steem_instance() + + @property + def settings(self) -> Dict[str, dict]: + """To ensure we always get fresh settings from the DB after a reload""" + return dict(((sym, c.settings) for sym, c in self.coins.items())) + + def load(self, tx_count=10000): + # Unlike other coins, it's important to load a lot of TXs, because many won't actually be transfers + # Thus the default TX count for Steem is 10,000 + self.tx_count = tx_count + for symbol, coin in self.coins.items(): + if not empty(coin.our_account): + continue + log.warning('The coin %s does not have `our_account` set. Refusing to load transactions.', coin) + del self.coins[symbol] + self.symbols = [s for s in self.symbols if s != symbol] + + def get_rpc(self, symbol): + """ + Returns a Steem instance for querying data and sending TXs. By default, uses the Beem shared_steem_instance. + + If a custom RPC list is specified in the Coin "custom json" settings, a new instance will be returned with the + RPCs specified in the json. + + :param symbol: Coin symbol to get Beem RPC instance for + :return beem.steem.Steem: An instance of :class:`beem.steem.Steem` for querying + """ + rpc_list = self.settings[symbol]['json'].get('rpcs') + return self.rpc if empty(rpc_list, itr=True) else Steem(node=rpc_list) + + def list_txs(self, batch=0) -> Generator[dict, None, None]: + if not self.loaded: + self.load() + for symbol, c in self.coins.items(): + acc_name = self.coins[symbol].our_account + acc = Account(acc_name, steem_instance=self.get_rpc(symbol)) + # get_account_history returns a generator with automatic batching, so we don't have to worry about batches. + txs = acc.get_account_history(-1, self.tx_count, only_ops=['transfer']) + yield from self.clean_txs(symbol=symbol, transactions=txs, account=acc_name) + + def clean_txs(self, symbol: str, transactions: Iterable[dict], account: str = None) -> Generator[dict, None, None]: + """ + Filters a list of transactions `transactions` as required, yields dict's conforming with :class:`models.Deposit` + + - Filters out transactions that are not marked as 'receive' + - Filters out mining transactions + - Filters by address if `account` is specified + - Filters out transactions that don't have enough confirms, and are not reported as 'trusted' + + :param symbol: Symbol of coin being cleaned + :param transactions: A ``list`` or generator producing dict's + :param account: If not None, only return TXs sent to this address. + :return Generator: A generator outputting dictionaries formatted as below + + Output Format:: + + { + txid:str, coin:str (symbol), vout:int, + tx_timestamp:datetime, address:str, amount:Decimal + } + + """ + + log.debug('Filtering transactions for %s', symbol) + for tx in transactions: + try: + t = self.clean_tx(tx, symbol, account) + if t is None: + continue + yield t + except (AttributeError, KeyError) as e: + log.warning('Steem TX missing important key? %s', str(e)) + except: + log.exception('Error filtering Steem TX, skipping... TX data: %s', tx) + + @staticmethod + def clean_tx(tx: dict, symbol: str, account: str, memo: str = None, memo_case: bool = False) -> Union[dict, None]: + """Filters an individual transaction. See :meth:`.clean_txs` for info""" + # log.debug(tx) + if tx.get('type', 'NOT SET') != 'transfer': + log.debug('Steem TX is not transfer. Type is: %s', tx.get('type', 'NOT SET')) + return None + + txid = tx.get('trx_id', None) + + _am = tx['amount'] # Transfer ops contain a dict 'amount', containing amount:int, nai:str, precision:int + + amt_sym = str(Asset(_am['nai']).symbol).upper() # Conv asset ID (e.g. @@000000021) to symbol, i.e. "STEEM" + + if amt_sym != symbol: # If the symbol doesn't match the symbol we were passed, skip this TX + return None + + # Convert integer amount/precision to Decimal's, preventing floating point issues + amt_int = Decimal(_am['amount']) + amt_prec = Decimal(_am['precision']) + + amt = amt_int / (Decimal(10) ** amt_prec) # Use precision value to convert from integer amt to decimal amt + + tx_memo = tx.get('memo') + + log.debug('Filtering/cleaning steem transaction, Amt: %f, TXID: %s', amt, txid) + + if tx['to'] != account or tx['from'] == account: + return None # If the transaction isn't to us (account), or it's from ourselves, ignore it. + if not empty(memo) and (tx_memo != memo or (not memo_case and tx_memo.lower() != memo.lower())): + return None + + d = parse(tx['timestamp']) + d = timezone.make_aware(d, pytz.UTC) + + return dict( + txid=txid, + coin=symbol, + vout=int(tx.get('op_in_trx', 0)), + tx_timestamp=d, + from_account=tx.get('from', None), + to_account=tx.get('to', None), + memo=tx_memo, + amount=Decimal(amt) + ) diff --git a/payments/coin_handlers/Steem/SteemManager.py b/payments/coin_handlers/Steem/SteemManager.py new file mode 100644 index 0000000..2446393 --- /dev/null +++ b/payments/coin_handlers/Steem/SteemManager.py @@ -0,0 +1,304 @@ +""" +**Copyright**:: + + +===================================================+ + | © 2019 Privex Inc. | + | https://www.privex.io | + +===================================================+ + | | + | CryptoToken Converter | + | | + | Core Developer(s): | + | | + | (+) Chris (@someguy123) [Privex] | + | | + +===================================================+ + +""" +import logging +from decimal import Decimal, getcontext, ROUND_DOWN +from typing import Tuple, List + +from beem.account import Account +from beem.asset import Asset +from beem.blockchain import Blockchain +from beem.exceptions import AccountDoesNotExistsException, MissingKeyError +from beem.steem import Steem +from beem.instance import shared_steem_instance + +from payments.coin_handlers import BaseManager +from payments.coin_handlers.Steem.SteemLoader import SteemLoader +from payments.coin_handlers.base import exceptions +from steemengine.helpers import empty + +log = logging.getLogger(__name__) +getcontext().rounding = ROUND_DOWN + + +class SteemManager(BaseManager): + """ + This class handles various operations for the **Steem** network, and supports both STEEM and SBD. + + It may or may not work with other Graphene coins, such as GOLOS / Whaleshares. + + It handles: + + - Validating source/destination accounts + - Checking the balance for a given account, as well as the total amount received with a certain ``memo`` + - Health checking + - Sending assets to users + + **Copyright**:: + + +===================================================+ + | © 2019 Privex Inc. | + | https://www.privex.io | + +===================================================+ + | | + | CryptoToken Converter | + | | + | Core Developer(s): | + | | + | (+) Chris (@someguy123) [Privex] | + | | + +===================================================+ + + """ + + provides = ["STEEM", "SBD"] # type: List[str] + """ + This attribute is automatically generated by scanning for :class:`models.Coin` s with the type ``steembase``. + This saves us from hard coding specific coin symbols. See __init__.py for populating code. + """ + + def __init__(self, symbol: str): + super().__init__(symbol) + settings = self.coin.settings['json'] + + rpcs = settings.get('rpcs') + # If you've specified custom RPC nodes in the custom JSON, make a new instance with those + # Otherwise, use the global shared_steem_instance. + self.rpc = shared_steem_instance() if empty(rpcs, itr=True) else Steem(rpcs) # type: Steem + self.rpc.set_password_storage(settings.get('pass_store', 'environment')) + # For easy reference, the Beem asset object, and precision + self.asset = asset = Asset(self.symbol) + self.precision = int(asset.precision) + + def health(self) -> Tuple[str, tuple, tuple]: + """ + Return health data for the passed symbol. + + Health data will include: 'Symbol', 'Status', 'Coin Name', 'API Node', 'Head Block', 'Block Time', + 'RPC Version', 'Our Account', 'Our Balance' (all strings) + + :return tuple health_data: (manager_name:str, headings:list/tuple, health_data:list/tuple,) + """ + headers = ('Symbol', 'Status', 'Coin Name', 'API Node', 'Head Block', 'Block Time', 'RPC Version', + 'Our Account', 'Our Balance') + + class_name = type(self).__name__ + api_node = asset_name = head_block = block_time = rpc_ver = our_account = balance = '' + + status = 'Okay' + try: + rpc = self.rpc + our_account = self.coin.our_account + if not self.address_valid(our_account): + status = 'Account {} not found'.format(our_account) + + asset_name = self.coin.display_name + balance = ('{0:,.' + str(self.precision) + 'f}').format(self.balance(our_account)) + api_node = rpc.rpc.url + props = rpc.get_dynamic_global_properties(use_stored_data=False) + head_block = str(props.get('head_block_number', '')) + block_time = props.get('time', '') + rpc_ver = rpc.get_blockchain_version() + except: + status = 'ERROR' + log.exception('Exception during %s.health for symbol %s', class_name, self.symbol) + + if status == 'Okay': + status = '{}'.format(status) + else: + status = '{}'.format(status) + + data = (self.symbol, status, asset_name, api_node, head_block, block_time, rpc_ver, our_account, balance) + return class_name, headers, data + + def health_test(self) -> bool: + """ + Check if our Steem node works or not, by requesting basic information such as the current block + time, and + checking if our sending/receiving account exists on Steem. + + :return bool: True if Steem appearsto be working, False if it seems to be broken. + """ + try: + _, _, health_data = self.health() + if 'Okay' in health_data[1]: + return True + return False + except: + return False + + def address_valid(self, address) -> bool: + """ + If an account exists on Steem, will return True. Otherwise False. + + :param address: Steem account to check existence of + :return bool: True if account exists, False if it doesn't + """ + try: + Account(address, steem_instance=self.rpc) + return True + except AccountDoesNotExistsException: + return False + + def get_deposit(self) -> tuple: + """ + Returns the deposit account for this symbol + + :return tuple: A tuple containing ('account', receiving_account). The memo must be generated + by the calling function. + """ + + return 'account', self.coin.our_account + + def balance(self, address: str = None, memo: str = None, memo_case: bool = False) -> Decimal: + """ + Get token balance for a given Steem account, if memo is given - get total symbol amt received with this memo. + + :param address: Steem account to get balance for, if not set, uses self.coin.our_account + :param memo: If not None, get total `self.symbol` received with this memo. + :param memo_case: Case sensitive memo search + :return: Decimal(balance) + """ + if not address: + address = self.coin.our_account + + acc = Account(address, steem_instance=self.rpc) + + if not empty(memo): + hist = acc.get_account_history(-1, 10000, only_ops=['transfer']) + total = Decimal(0) + for h in hist: + tx = SteemLoader.clean_tx(h, self.symbol, address, memo) + if tx is None: + continue + total += tx['amount'] + return total + + bal = acc.get_balance('available', self.symbol) + return Decimal(bal.amount) + + def find_steem_tx(self, tx_data, last_blocks=15): + """ + Used internally to get the transaction ID after a transaction has been broadcasted + + :param dict tx_data: Transaction data returned by a beem broadcast operation, must include 'signatures' + :param int last_blocks: Amount of previous blocks to search for the transaction + :return dict: Transaction data from the blockchain {transaction_id, ref_block_num, ref_block_prefix, + expiration, operations, extensions, signatures, block_num, transaction_num} + + :return None: If the transaction wasn't found, None will be returned. + """ + # Code taken/based from @holgern/beem blockchain.py + chain = Blockchain(steem_instance=self.rpc, mode='head') + current_num = chain.get_current_block_num() + for block in chain.blocks(start=current_num - last_blocks, stop=current_num + 5): + for tx in block.transactions: + if sorted(tx["signatures"]) == sorted(tx_data["signatures"]): + return tx + return None + + def send(self, amount: Decimal, address: str, from_address: str = None, memo: str = None) -> dict: + """ + Send a supported currency to a given address/account, optionally specifying a memo if supported + + Example - send 1.23 STEEM from @someguy123 to @privex with memo 'hello' + + >>> s = SteemManager('STEEM') + >>> s.send(from_address='someguy123', address='privex', amount=Decimal('1.23'), memo='hello') + + :param Decimal amount: Amount of currency to send, as a Decimal() + :param address: Account to send the currency to + :param from_address: Account to send the currency from + :param memo: Memo to send currency with + :raises AttributeError: When both `from_address` and `self.coin.our_account` are blank. + :raises ArithmeticError: When the amount is lower than the lowest amount allowed by the asset's precision + :raises AuthorityMissing: Cannot send because we don't have authority to (missing key etc.) + :raises AccountNotFound: The requested account doesn't exist + :raises NotEnoughBalance: The account `from_address` does not have enough balance to send this amount. + :return dict: Result Information + + Format:: + + { + txid:str - Transaction ID - None if not known, + coin:str - Symbol that was sent, + amount:Decimal - The amount that was sent (after fees), + fee:Decimal - TX Fee that was taken from the amount, + from:str - The account/address the coins were sent from, + send_type:str - Should be statically set to "send" + } + + """ + + # Try from_address first. If that's empty, try using self.coin.our_account. If both are empty, abort. + if empty(from_address): + if empty(self.coin.our_account): + raise AttributeError("Both 'from_address' and 'coin.our_account' are empty. Cannot send.") + from_address = self.coin.our_account + + prec = self.precision + sym = self.symbol + memo = "" if empty(memo) else memo + + try: + if type(amount) != Decimal: + if type(amount) == float: + amount = ('{0:.' + str(self.precision) + 'f}').format(amount) + amount = Decimal(amount) + ### + # Various sanity checks, e.g. checking amount is valid, to/from account are valid, we have + # enough balance to send this amt, etc. + ### + if amount < Decimal(pow(10, -prec)): + log.warning('Amount %s was passed, but is lower than precision for %s', amount, sym) + raise ArithmeticError('Amount {} is lower than token {}s precision of {} DP'.format(amount, sym, prec)) + + acc = Account(from_address, steem_instance=self.rpc) + + if not self.address_valid(address): raise exceptions.AccountNotFound('Destination account does not exist') + if not self.address_valid(from_address): raise exceptions.AccountNotFound('From account does not exist') + + bal = self.balance(from_address) + if bal < amount: + raise exceptions.NotEnoughBalance( + 'Account {} has balance {} but needs {} to send this tx'.format(from_address, bal, amount) + ) + + ### + # Broadcast the transfer transaction on the network, and return the necessary data + ### + log.debug('Sending %f %s to @%s', amount, sym, address) + tfr = acc.transfer(address, amount, sym, memo) + # Beem's finalizeOp doesn't include TXID, so we try to find the TX on the blockchain after broadcast + tx = self.find_steem_tx(tfr) + + log.debug('Success? TX Data - Transfer: %s Lookup TX: %s', tfr, tx) + # Return TX data compatible with BaseManager standard + return { + # There's a risk we can't get the TXID, and so we fall back to None. + 'txid': tx.get('transaction_id', None), + 'coin': sym, + 'amount': amount, + 'fee': Decimal(0), + 'from': from_address, + 'send_type': 'send' + } + except MissingKeyError: + raise exceptions.AuthorityMissing('Missing active key for sending account {}'.format(from_address)) + + + + diff --git a/payments/coin_handlers/Steem/__init__.py b/payments/coin_handlers/Steem/__init__.py new file mode 100644 index 0000000..4be96d2 --- /dev/null +++ b/payments/coin_handlers/Steem/__init__.py @@ -0,0 +1,99 @@ +""" +**Steem Coin Handler** + +This python module is a **Coin Handler** for Privex's CryptoToken Converter, designed to handle all required +functionality for both receiving and sending tokens on the **Steem** network. + +It will automatically handle any :class:`payments.models.Coin` which has it's type set to ``steembase`` + +**Coin object settings**: + + For each :class:`payments.models.Coin` you intend to use with this handler, you should configure it as such: + + ============= ================================================================================================== + Coin Key Description + ============= ================================================================================================== + coin_type This should be set to ``Steem Network (or compatible fork)`` (db value: steembase) + our_account This should be set to the username of the account you want to use for receiving/sending + setting_json A JSON string for optional extra config (see below) + ============= ================================================================================================== + + Extra JSON (Handler Custom) config options: + + - ``rpcs`` - A JSON list of RPC nodes to use, with a full HTTP/HTTPS URL. If this is not specified, Beem + will automatically try to use the best available RPC node for the Steem network. + - ``pass_store`` - Generally you do not need to touch this. It controls where Beem will look for the wallet + password. It defaults to ``environment`` + + Example JSON custom config:: + + { + "rpcs": [ + "https://steemd.privex.io", + "https://api.steemit.com", + "https://api.steem.house" + ], + "pass_store": "environment" + } + + + +**Copyright**:: + + +===================================================+ + | © 2019 Privex Inc. | + | https://www.privex.io | + +===================================================+ + | | + | CryptoToken Converter | + | | + | Core Developer(s): | + | | + | (+) Chris (@someguy123) [Privex] | + | | + +===================================================+ + +""" +from payments.coin_handlers.Steem.SteemLoader import SteemLoader +from payments.coin_handlers.Steem.SteemManager import SteemManager +from django.conf import settings +import logging +from payments.models import Coin + +log = logging.getLogger(__name__) + +loaded = False + + +def reload(): + """ + Reload's the ``provides`` property for the loader and manager from the DB. + + By default, since new Steem forks are constantly being created, our classes can provide for any + :class:`models.Coin` by scanning for coins with the type ``steembase``. This saves us from hard coding + specific coin symbols. + """ + # Set loaded to True, so we aren't constantly reloading the ``provides``, only when we need to. + global loaded + loaded = True + + log.debug('Checking if steembase is in COIN_TYPES') + if 'steembase' not in dict(settings.COIN_TYPES): + log.debug('steembase not in COIN_TYPES, adding it.') + settings.COIN_TYPES += (('steembase', 'Steem Network (or compatible fork)',),) + + # Grab a simple list of coin symbols with the type 'bitcoind' to populate the provides lists. + provides = Coin.objects.filter(coin_type='steembase').values_list('symbol', flat=True) + SteemLoader.provides = provides + SteemManager.provides = provides + + +# Only run the initialisation code once. +# After the first run, reload() will be called only when there's a change by the coin handler system +if not loaded: + reload() + +exports = { + "loader": SteemLoader, + "manager": SteemManager +} diff --git a/payments/coin_handlers/SteemEngine/SteemEngineLoader.py b/payments/coin_handlers/SteemEngine/SteemEngineLoader.py index b8cdd87..785aa90 100644 --- a/payments/coin_handlers/SteemEngine/SteemEngineLoader.py +++ b/payments/coin_handlers/SteemEngine/SteemEngineLoader.py @@ -17,7 +17,7 @@ """ import logging -from decimal import Decimal +from decimal import Decimal, getcontext, ROUND_DOWN from time import sleep from typing import Generator, Iterable, List @@ -105,6 +105,7 @@ def clean_txs(self, account: str, symbol: str, transactions: Iterable[dict]) -> if tx['to'].lower() != account.lower(): continue # If we aren't the receiver, we don't need it. # Cache the token for 5 mins, so we aren't spamming the token API token = cache.get_or_set('stmeng:'+symbol, lambda: self.eng_rpc.get_token(symbol), 300) + q = tx['quantity'] if type(q) == float: q = ('{0:.' + str(token['precision']) + 'f}').format(tx['quantity']) diff --git a/payments/coin_handlers/SteemEngine/SteemEngineManager.py b/payments/coin_handlers/SteemEngine/SteemEngineManager.py index 86c6560..6aa5646 100644 --- a/payments/coin_handlers/SteemEngine/SteemEngineManager.py +++ b/payments/coin_handlers/SteemEngine/SteemEngineManager.py @@ -26,6 +26,7 @@ getcontext().rounding = ROUND_DOWN + log = logging.getLogger(__name__) @@ -108,7 +109,7 @@ def health(self) -> Tuple[str, tuple, tuple]: data = (self.symbol, status, api_node, token_name, issuer, precision, our_account, balance) return class_name, headers, data - def health_test(self): + def health_test(self) -> bool: """ Check if the SteemEngine API and Steem node works or not, by requesting basic information such as the token metadata, and checking if our sending/receiving account exists on Steem. @@ -199,6 +200,7 @@ def issue(self, amount: Decimal, address: str, memo: str = None) -> dict: try: token = self.eng_rpc.get_token(symbol=self.symbol) + # If we get passed a float for some reason, make sure we trim it to the token's precision before # converting it to a Decimal. if type(amount) == float: @@ -264,6 +266,7 @@ def send(self, amount, address, memo=None, from_address=None) -> dict: from_address = self.coin.our_account try: token = self.eng_rpc.get_token(symbol=self.symbol) + # If we get passed a float for some reason, make sure we trim it to the token's precision before # converting it to a Decimal. if type(amount) == float: diff --git a/payments/coin_handlers/base/BaseLoader.py b/payments/coin_handlers/base/BaseLoader.py index 13cf270..e427de4 100644 --- a/payments/coin_handlers/base/BaseLoader.py +++ b/payments/coin_handlers/base/BaseLoader.py @@ -1,6 +1,6 @@ import logging from abc import ABC, abstractmethod -from typing import Generator +from typing import Generator, Dict from django.conf import settings from payments.models import Coin @@ -70,7 +70,7 @@ def __init__(self, symbols: list = None): # self.coins is a dictionary mapping symbols to their Coin objects, for easy lookup. # e.g. self.coins['BTC'].display_name coins = Coin.objects.filter(symbol__in=symbols, enabled=True) - self.coins = {c.symbol: c for c in coins} + self.coins = {c.symbol: c for c in coins} # type: Dict[str, Coin] self.symbols = self.coins.keys() # For your convenience, self.transactions is pre-defined as a list, for loading into by your functions. diff --git a/payments/management/commands/load_txs.py b/payments/management/commands/load_txs.py index b1fd05a..0738841 100644 --- a/payments/management/commands/load_txs.py +++ b/payments/management/commands/load_txs.py @@ -45,7 +45,7 @@ def load_txs(self, symbol): for l in loaders: # type: BaseLoader log.debug('Scanning using loader %s', type(l)) finished = False - l.load(1000) + l.load() txs = l.list_txs(self.BATCH) while not finished: log.debug('Loading batch of %s TXs for DB insert', self.BATCH) @@ -75,7 +75,8 @@ def import_batch(self, txs: iter, batch: int) -> bool: continue log.debug('Storing TX %s', tx['txid']) tx['coin'] = Coin.objects.get(symbol=tx['coin']) - Deposit(**tx).save() + with transaction.atomic(): + Deposit(**tx).save() except: log.exception('Error saving TX %s for coin %s, will skip.', tx['txid'], tx['coin']) finally: diff --git a/payments/migrations/0003_auto_20190403_0338.py b/payments/migrations/0003_auto_20190403_0338.py new file mode 100644 index 0000000..6bb6476 --- /dev/null +++ b/payments/migrations/0003_auto_20190403_0338.py @@ -0,0 +1,68 @@ +# Generated by Django 2.1.7 on 2019-04-03 03:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('payments', '0002_coin_lowfunds_20190401'), + ] + + operations = [ + migrations.AlterField( + model_name='addressaccountmap', + name='deposit_address', + field=models.CharField(max_length=255, verbose_name='Deposit Address / Account'), + ), + migrations.AlterField( + model_name='addressaccountmap', + name='deposit_memo', + field=models.CharField(blank=True, default=None, max_length=1000, null=True, verbose_name='Deposit Memo (if required)'), + ), + migrations.AlterField( + model_name='addressaccountmap', + name='destination_address', + field=models.CharField(max_length=255, verbose_name='Destination Address / Account'), + ), + migrations.AlterField( + model_name='addressaccountmap', + name='destination_memo', + field=models.CharField(blank=True, default=None, max_length=1000, null=True, verbose_name='Destination Memo (if required)'), + ), + migrations.AlterField( + model_name='conversion', + name='to_memo', + field=models.CharField(blank=True, max_length=1000, null=True, verbose_name='Destination Memo (if applicable)'), + ), + migrations.AlterField( + model_name='deposit', + name='address', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='deposit', + name='convert_dest_memo', + field=models.CharField(blank=True, max_length=1000, null=True), + ), + migrations.AlterField( + model_name='deposit', + name='from_account', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='deposit', + name='memo', + field=models.CharField(blank=True, max_length=1000, null=True), + ), + migrations.AlterField( + model_name='deposit', + name='refund_memo', + field=models.CharField(blank=True, max_length=1000, null=True), + ), + migrations.AlterField( + model_name='deposit', + name='to_account', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/payments/models.py b/payments/models.py index 0d1ed42..981af6b 100644 --- a/payments/models.py +++ b/payments/models.py @@ -196,13 +196,13 @@ class AddressAccountMap(models.Model): # The `deposit_address` can be a crypto address, or an account name, depending on what the coin requires. # If you use an account name, you should specify `deposit_memo` for detecting the transaction deposit_coin = models.ForeignKey(Coin, db_index=True, on_delete=models.DO_NOTHING, related_name='deposit_maps') - deposit_address = models.CharField('Deposit Address / Account', max_length=100) - deposit_memo = models.CharField('Deposit Memo (if required)', max_length=255, + deposit_address = models.CharField('Deposit Address / Account', max_length=255) + deposit_memo = models.CharField('Deposit Memo (if required)', max_length=1000, blank=True, null=True, default=None) # The `destination_*` fields define the crypto/token that the deposited crypto/token will be converted into. destination_coin = models.ForeignKey(Coin, db_index=True, on_delete=models.DO_NOTHING, related_name='dest_maps') - destination_address = models.CharField('Destination Address / Account', max_length=100) - destination_memo = models.CharField('Destination Memo (if required)', max_length=255, + destination_address = models.CharField('Destination Address / Account', max_length=255) + destination_memo = models.CharField('Destination Memo (if required)', max_length=1000, blank=True, null=True, default=None) @property @@ -278,20 +278,20 @@ class Deposit(models.Model): tx_timestamp = models.DateTimeField('Transaction Date/Time', blank=True, null=True) """The date/time the transaction actually occurred on the chain""" - address = models.CharField(max_length=100, blank=True, null=True) + address = models.CharField(max_length=255, blank=True, null=True) """If the deposit is from a classic Bitcoin-like cryptocurrency with addresses, then you should enter the address where the coins were deposited into, in this field.""" # If the deposit is from a Bitshares-like cryptocurrency (Steem, GOLOS, EOS), then you should enter the # sending account into `from_account`, our receiving account into `to_account`, and memo into `memo` # Tokens received on Steem Engine should use these fields. - from_account = models.CharField(max_length=100, blank=True, null=True) + from_account = models.CharField(max_length=255, blank=True, null=True) """If account-based coin, contains the name of the account that sent the coins""" - to_account = models.CharField(max_length=100, blank=True, null=True) + to_account = models.CharField(max_length=255, blank=True, null=True) """If account-based coin, contains the name of the account that the coins were deposited into""" - memo = models.CharField(max_length=255, blank=True, null=True) + memo = models.CharField(max_length=1000, blank=True, null=True) """If the coin supports memos, and they're required to identify a deposit, use this field.""" amount = models.DecimalField(max_digits=MAX_STORED_DIGITS, decimal_places=MAX_STORED_DP) @@ -305,13 +305,13 @@ class Deposit(models.Model): convert_dest_address = models.CharField(max_length=255, null=True, blank=True) """The destination address. Set after a deposit has been analyzed, and we know what coin it will be converted to.""" - convert_dest_memo = models.CharField(max_length=255, null=True, blank=True) + convert_dest_memo = models.CharField(max_length=1000, null=True, blank=True) """The destination memo. Set after a deposit has been analyzed, and we know what coin it will be converted to.""" # If something goes wrong with this transaction, and it was refunded, then we store # all of the refund details, for future reference. refund_address = models.CharField('Refunded to this account/address', max_length=500, blank=True, null=True) - refund_memo = models.CharField(max_length=500, blank=True, null=True) + refund_memo = models.CharField(max_length=1000, blank=True, null=True) refund_coin = models.CharField('The coin (symbol) that was refunded to them', max_length=10, blank=True, null=True) refund_amount = models.DecimalField(max_digits=MAX_STORED_DIGITS, decimal_places=MAX_STORED_DP, default=0) refund_txid = models.CharField(max_length=500, blank=True, null=True) @@ -363,7 +363,7 @@ class Conversion(models.Model): to_address = models.CharField('Destination Address / Account', max_length=255) """Where was it sent to?""" - to_memo = models.CharField('Destination Memo (if applicable)', max_length=255, blank=True, null=True) + to_memo = models.CharField('Destination Memo (if applicable)', max_length=1000, blank=True, null=True) to_amount = models.DecimalField('Amount Sent', max_digits=MAX_STORED_DIGITS, decimal_places=MAX_STORED_DP) """The amount of ``to_coin`` that was sent, stored as a high precision Decimal""" diff --git a/steemengine/settings/custom.py b/steemengine/settings/custom.py index 725569e..1eb9f96 100644 --- a/steemengine/settings/custom.py +++ b/steemengine/settings/custom.py @@ -64,14 +64,13 @@ # Load coin handlers from this absolute module path COIN_HANDLERS_BASE = env('COIN_HANDLERS_BASE', 'payments.coin_handlers') -COIN_HANDLERS = env('COIN_HANDLERS', 'SteemEngine,Bitcoin').split(',') # A comma separated list of modules to load +# A comma separated list of modules to load +COIN_HANDLERS = env('COIN_HANDLERS', 'SteemEngine,Bitcoin,Steem').split(',') # After the first email to inform admins a wallet is low, how long before we send out a second notification? # (in hours) (Default: 12 hrs) LOWFUNDS_RENOTIFY = int(env('LOWFUNDS_RENOTIFY', 12)) - - ######### # Defaults for pre-installed Coin Handlers, to avoid potential exceptions when accessing their settings. ####