support for EVM, substrate, integration with chainlist and cosmos.dir…

…ectory (#4)

* add substrate portfolio
* fix balance on substrate
* update README, remove decimals, symbol, demom from config file
* update to support other evm chains

README.md

@@ -20,8 +20,13 @@ pip install -r requirements.txt
 
 ## Edit config.yaml
 
-> type is currently not used. In the future, it will be used for other chain (ETH, DOT, ...)
+Supported types of wallet: cosmos, evm, substrate (polkadot for example)
 
+For cosmos wallets (celestia for example), the name of the network must match what is in https://chains.cosmos.directory
+
+For evm wallets, if it is an erc20 token, you need to specify the contract address in the field contract_address in the wallet.
+
+Replace the api, rpc accordingly (for example, use https://moonbeam.public.blastapi.io if you use the moonbeam evm)
 
 ## Run it
 
@@ -43,10 +48,17 @@ sudo systemctl start wallet-exporter
 # Test it
 
 ```
+Example
 curl -s localhost:9877/metric
 
-account_info{address="cosmos1gswfh88s88s2evtsutwt8heh59jttjglhdlwtwj",name="validator",network="cosmos", type="balance"} 54.451031
-account_info{address="cosmos1r59ugu6w72s88s2evtsutwt8heh59jpqp9mm3ew",name="delegator",network="cosmos", type="balance"} 25600.995009
+account_info{address="cosmos1gswfh88s88s2evtsutwt8heh59jttjglhdlwtwj",name="validator",network="cosmoshub", type="balance"} 54.451031
+account_info{address="cosmos1gswfh88s88s2evtsutwt8heh59jttjglhdlwtwj",name="validator",network="cosmoshub", type="delegations"} 0.0
+account_info{address="cosmos1gswfh88s88s2evtsutwt8heh59jttjglhdlwtwj",name="validator",network="cosmoshub", type="unbounding_delegations"} 0.0
+account_info{address="cosmos1gswfh88s88s2evtsutwt8heh59jttjglhdlwtwj",name="validator",network="cosmoshub", type="rewards"} 0.0
+account_info{address="0x95222290DD7278Aa3Ddd389Cc1E1d165CC4BAfe9",name="broadcaster ethereum",network="ethereum",type="balance"} 6.388107948244274
+account_info{address="0x95222290DD7278Aa3Ddd389Cc1E1d165CC4BAfe9",name="broadcaster matic",network="ethereum",type="balance"} 0.4753891567500353
+account_info{address="1FwzEXsZedfWFPGtJ3Ex8SFLhvugrA9aJN9GL1GeHpYeqf7",name="broadcaster polkadot",network="polkadot",type="balance"} 46000.0
+
 ```
 # TODO
 

config.yaml

@@ -1,13 +1,35 @@
 networks:
-- name: cosmos
+- name: cosmoshub
   rpc: http://127.0.0.1:26657
   api: http://127.0.0.1:1317
-  decimals: 6
-  denom: uatom
-  symbol: ATOM
   type: cosmos
   wallets:
   - name: validator
     address: cosmos1gswfh88s88s2evtsutwt8heh59jttjglhdlwtwj
   - name: delegator
-    address: cosmos1r59ugu6w72s88s2evtsutwt8heh59jpqp9mm3ew
+    address: cosmos1r59ugu6w72s88s2evtsutwt8heh59jpqp9mm3ew
+- name: ethereum
+  rpc: http://127.0.0.1:26657
+  api: http://127.0.0.1:1317
+  type: evm
+  wallets:
+  - name: broadcaster ethereum
+    address: "0x95222290DD7278Aa3Ddd389Cc1E1d165CC4BAfe9"
+  - name: broadcaster matic
+    address: "0x95222290DD7278Aa3Ddd389Cc1E1d165CC4BAfe9"
+    contract_address: "0x7D1AfA7B718fb893dB30A3aBc0Cfc608AaCfeBB0"
+- name: moonbeam
+  rpc: http://127.0.0.1:26657
+  api: http://127.0.0.1:1317
+  type: evm
+  wallets:
+  - name: broadcaster moonbeam
+    address: "0xDd4273d24eAe0141F47061b08c5c73668C975A25"
+- name: polkadot
+  rpc: wss://127.0.0.1:26657
+  api: wss://127.0.0.1:1317
+  type: substrate
+  wallets:
+  - name: broadcaster polkadot
+    address: "1FwzEXsZedfWFPGtJ3Ex8SFLhvugrA9aJN9GL1GeHpYeqf7"
+

cosmos.py

@@ -103,3 +103,17 @@ def get_rewards(apiprovider, addr: str, maindenom, rpc_call_status_counter):
         return 0
     except Exception as addr_balancer_err:
         raise addr_balancer_err
+
+
+def get_cosmos_registry(rpc_call_status_counter):
+    try:
+        params: dict = {}
+        d = http_json_call(
+            url="https://chains.cosmos.directory",
+            rpc_call_status_counter=rpc_call_status_counter,
+            params=params,
+        )
+
+        return d["chains"]
+    except Exception as err:
+        raise err

ethereum.py

@@ -1,14 +1,35 @@
 from web3 import Web3
+from utils import http_json_call
+
 from metrics_enum import MetricsUrlStatus
 
 
-def get_ethereum_balance(apiprovider, wallet, rpc_call_status_counter):
+def get_evm_chains_data(rpc_call_status_counter):
+    try:
+        d = http_json_call(
+            url="https://chainid.network/chains.json",
+            rpc_call_status_counter=rpc_call_status_counter,
+            params={},
+        )
+        return d
+    except Exception as err:
+        raise err
+
+
+def get_chain_symbol(chain_id, chain_data):
+    for chain in chain_data:
+        if chain["chainId"] == chain_id:
+            return chain["nativeCurrency"]["symbol"]
+    return "Unknown"
+
+
+def get_ethereum_balance(apiprovider, wallet, rpc_call_status_counter, chains_evm):
     try:
         addr = wallet["address"]
         # if it is erc20
         if "contract_address" in wallet:
             contract_address = wallet["contract_address"]
-            erc20_balance = get_erc20_balance(
+            erc20_data = get_erc20_balance(
                 apiprovider=apiprovider,
                 addr=addr,
                 contract_address=contract_address,
@@ -17,15 +38,17 @@ def get_ethereum_balance(apiprovider, wallet, rpc_call_status_counter):
             rpc_call_status_counter.labels(
                 url=apiprovider, status=MetricsUrlStatus.SUCCESS.value
             ).inc()
-            return erc20_balance
+            return {"balance": erc20_data["balance"], "symbol": erc20_data["symbol"]}
         else:
             web3 = Web3(Web3.HTTPProvider(apiprovider))
             balance = web3.eth.get_balance(addr)
             balance_ether = web3.from_wei(balance, "ether")
+            chain_id = web3.eth.chain_id
+            symbol = get_chain_symbol(chain_id=chain_id, chain_data=chains_evm)
             rpc_call_status_counter.labels(
                 url=apiprovider, status=MetricsUrlStatus.SUCCESS.value
             ).inc()
-            return balance_ether
+            return {"balance": balance_ether, "symbol": symbol}
     except Exception as addr_balancer_err:
         rpc_call_status_counter.labels(
             url=apiprovider, status=MetricsUrlStatus.FAILED.value
@@ -53,13 +76,21 @@ def get_erc20_balance(
             "outputs": [{"name": "", "type": "uint8"}],
             "type": "function",
         },
+        {
+            "constant": True,
+            "inputs": [],
+            "name": "symbol",
+            "outputs": [{"name": "", "type": "string"}],
+            "type": "function",
+        },
     ]
     web3 = Web3(Web3.HTTPProvider(apiprovider))
     contract = web3.eth.contract(address=contract_address, abi=minABI)
     balance = contract.functions.balanceOf(addr).call()
     decimals = contract.functions.decimals().call()
+    symbol = contract.functions.symbol().call()
     adjusted_balance = balance / (10**decimals)
     rpc_call_status_counter.labels(
         url=apiprovider, status=MetricsUrlStatus.SUCCESS.value
     ).inc()
-    return adjusted_balance
+    return {"balance": adjusted_balance, "symbol": symbol}

exporter.py

@@ -1,19 +1,23 @@
 """Application exporter"""
 
+import argparse
 import os
 import time
-from prometheus_client import start_http_server, Gauge, Counter
-import argparse
+
 from dotenv import load_dotenv
-from utils import read_config_file, configure_logging
+from prometheus_client import Counter, Gauge, start_http_server
+
 from cosmos import (
-    get_maincoin_balance,
+    get_cosmos_registry,
     get_delegations,
-    get_unbonding_delegations,
+    get_maincoin_balance,
     get_rewards,
+    get_unbonding_delegations,
 )
+from ethereum import get_evm_chains_data, get_ethereum_balance
 from metrics_enum import MetricsAccountInfo, NetworkType
-from ethereum import get_ethereum_balance, get_erc20_balance
+from substrate import get_substrate_account_balance
+from utils import configure_logging, read_config_file
 
 
 class AppMetrics:
@@ -40,6 +44,8 @@ def __init__(self, polling_interval_seconds=60, walletconfig=False, logging=Fals
             "Count the number of success or failed http call for a given url",
             ["url", "status"],
         )
+        self.cosmos_registry = get_cosmos_registry(self.rpc_call_status_counter)
+        self.chains_evm = get_evm_chains_data(self.rpc_call_status_counter)
 
         logging.debug(walletconfig)
 
@@ -50,7 +56,7 @@ def run_metrics_loop(self):
             self.fetch()
             time.sleep(self.polling_interval_seconds)
 
-    def fetch_balance(self, network, wallet):
+    def fetch_balance(self, network, wallet, chain_registry):
         network_name = network["name"]
         network_type = network["type"]
         balance = 0
@@ -60,21 +66,37 @@ def fetch_balance(self, network, wallet):
                 get_maincoin_balance(
                     network["api"],
                     wallet["address"],
-                    network["denom"],
+                    chain_registry["denom"],
                     self.rpc_call_status_counter,
                 )
-            ) / (10 ** network["decimals"])
-            self.logging.info(f"{wallet['address']} has {balance} {network['symbol']}")
-        elif network_type == NetworkType.ETHEREUM.value:
-            balance = get_ethereum_balance(
+            ) / (10 ** chain_registry["decimals"])
+            self.logging.info(
+                f"{wallet['address']} has {balance} {chain_registry['symbol']}"
+            )
+        elif network_type == NetworkType.EVM.value:
+            balance_data = get_ethereum_balance(
                 apiprovider=network["api"],
                 wallet=wallet,
                 rpc_call_status_counter=self.rpc_call_status_counter,
+                chains_evm=self.chains_evm,
             )
+            balance = balance_data["balance"]
+            symbol = balance_data["symbol"]
             if "contract_address" in wallet:
-                self.logging.info(f"{wallet['address']} has {balance} {wallet['symbol']}")
+                self.logging.info(f"{wallet['address']} has {balance} {symbol}")
             else:
-                self.logging.info(f"{wallet['address']} has {balance} {network['symbol']}")
+                self.logging.info(f"{wallet['address']} has {balance} {symbol}")
+        elif network_type == NetworkType.SUBSTRATE.value:
+            substrate_info = get_substrate_account_balance(
+                node_url=network["api"],
+                address=wallet["address"],
+                rpc_call_status_counter=self.rpc_call_status_counter,
+            )
+            balance = substrate_info.get("balance") / 10 ** substrate_info.get(
+                "decimals"
+            )
+            symbol = substrate_info.get("symbol")
+            self.logging.info(f"{wallet['address']} has {balance} {symbol}")
 
         self.account_info.labels(
             network=network_name,
@@ -83,7 +105,7 @@ def fetch_balance(self, network, wallet):
             type=MetricsAccountInfo.BALANCE.value,
         ).set(balance)
 
-    def fetch_delegations(self, network, wallet):
+    def fetch_delegations(self, network, wallet, chain_registry):
         network_name = network["name"]
         network_type = network["type"]
 
@@ -92,10 +114,10 @@ def fetch_delegations(self, network, wallet):
                 get_delegations(
                     network["api"],
                     wallet["address"],
-                    network["denom"],
+                    chain_registry["denom"],
                     self.rpc_call_status_counter,
                 )
-            ) / (10 ** network["decimals"])
+            ) / (10 ** chain_registry["decimals"])
             self.logging.info(f"{wallet['address']} has {delegations} delegations")
             self.account_info.labels(
                 network=network_name,
@@ -104,7 +126,7 @@ def fetch_delegations(self, network, wallet):
                 type=MetricsAccountInfo.DELEGATIONS.value,
             ).set(delegations)
 
-    def fetch_unbounding_delegations(self, network, wallet):
+    def fetch_unbounding_delegations(self, network, wallet, chain_registry):
         network_name = network["name"]
         network_type = network["type"]
 
@@ -115,7 +137,7 @@ def fetch_unbounding_delegations(self, network, wallet):
                     wallet["address"],
                     self.rpc_call_status_counter,
                 )
-            ) / (10 ** network["decimals"])
+            ) / (10 ** chain_registry["decimals"])
             self.logging.info(
                 f"{wallet['address']} has {unbounding_delegations} unbounding delegations"
             )
@@ -126,7 +148,7 @@ def fetch_unbounding_delegations(self, network, wallet):
                 type=MetricsAccountInfo.UNBOUNDING_DELEGATIONS.value,
             ).set(unbounding_delegations)
 
-    def fetch_rewards(self, network, wallet):
+    def fetch_rewards(self, network, wallet, chain_registry):
         network_name = network["name"]
         network_type = network["type"]
 
@@ -135,10 +157,10 @@ def fetch_rewards(self, network, wallet):
                 get_rewards(
                     network["api"],
                     wallet["address"],
-                    network["denom"],
+                    chain_registry["denom"],
                     self.rpc_call_status_counter,
                 )
-            ) / (10 ** network["decimals"])
+            ) / (10 ** chain_registry["decimals"])
             self.logging.info(f"{wallet['address']} has {rewards} rewards")
             self.account_info.labels(
                 network=network_name,
@@ -157,14 +179,34 @@ def fetch(self):
 
         for network in self.walletconfig["networks"]:
             self.logging.debug(network)
+
+            chain_registry = None
+            if network["type"] == NetworkType.COSMOS.value:
+                for chain in self.cosmos_registry:
+                    if chain["name"] == network["name"]:
+                        chain_registry = chain
+                        break
+                if chain_registry is None:
+                    self.logging.error(
+                        f"Cannot find chain {network} in cosmos registry"
+                    )
+                    continue
+
             for wallet in network["wallets"]:
                 try:
                     self.logging.info(f"Fetching {wallet['address']}")
-                    self.fetch_balance(network=network, wallet=wallet)
-                    self.fetch_delegations(network=network, wallet=wallet)
-                    self.fetch_unbounding_delegations(network=network, wallet=wallet)
-                    self.fetch_rewards(network=network, wallet=wallet)
-
+                    self.fetch_balance(
+                        network=network, wallet=wallet, chain_registry=chain_registry
+                    )
+                    self.fetch_delegations(
+                        network=network, wallet=wallet, chain_registry=chain_registry
+                    )
+                    self.fetch_unbounding_delegations(
+                        network=network, wallet=wallet, chain_registry=chain_registry
+                    )
+                    self.fetch_rewards(
+                        network=network, wallet=wallet, chain_registry=chain_registry
+                    )
                 except Exception as e:
                     self.logging.error(str(e))
 
@@ -225,7 +267,7 @@ def main():
 
     log.debug(messages)
 
-    log.debug(f"Loading .env")
+    log.debug("Loading .env")
     load_dotenv()  # take environment variables from .env
     polling_interval_seconds = int(os.getenv("POLLING_INTERVAL_SECONDS", "60"))
     exporter_port = int(os.getenv("EXPORTER_PORT", "9877"))

metrics_enum.py

@@ -15,4 +15,5 @@ class MetricsAccountInfo(Enum):
 
 class NetworkType(Enum):
     COSMOS = "cosmos"
-    ETHEREUM = "ethereum"
+    EVM = "evm"
+    SUBSTRATE = "substrate"

requirements.txt

@@ -3,4 +3,6 @@ requests
 python-dotenv
 pyyaml
 structlog
-web3
+web3
+polkadot
+substrate-interface