Merge pull request #327 from InjectiveLabs/feat/load_official_tokens_…

…list

Feat/load official tokens list

CHANGELOG.md

@@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file.
 - Support for all queries in the "IBC Channel" module
 - Support for all queries in the "IBC Client" module
 - Support for all queries in the "IBC Connection" module
+- Tokens initialization from the official tokens list in https://github.com/InjectiveLabs/injective-lists
 
 ### Changed
 - Refactored cookies management logic to use all gRPC calls' responses to update the current cookies

README.md

@@ -16,7 +16,7 @@ sudo dnf install python3-devel autoconf automake gcc gcc-c++ libffi-devel libtoo
 **macOS**
 
 ```bash
-brew install autoconf automake libtool
+brew install autoconf automake libtool bufbuild/buf/buf
 ```
 
 ### Quick Start
@@ -67,7 +67,7 @@ Upgrade `pip` to the latest version, if you see these warnings:
 
 3. Fetch latest denom config
 ```
-poetry run python pyinjective/fetch_metadata.py
+poetry run python pyinjective/utils/fetch_metadata.py
 ```
 
 Note that the [sync client](https://github.com/InjectiveLabs/sdk-python/blob/master/pyinjective/client.py) has been deprecated as of April 18, 2022. If you are using the sync client please make sure to transition to the [async client](https://github.com/InjectiveLabs/sdk-python/blob/master/pyinjective/async_client.py), for more information read [here](https://github.com/InjectiveLabs/sdk-python/issues/101)

pyinjective/async_client.py

@@ -44,6 +44,7 @@
 from pyinjective.core.network import Network
 from pyinjective.core.tendermint.grpc.tendermint_grpc_api import TendermintGrpcApi
 from pyinjective.core.token import Token
+from pyinjective.core.tokens_file_loader import TokensFileLoader
 from pyinjective.core.tx.grpc.tx_grpc_api import TxGrpcApi
 from pyinjective.exceptions import NotFoundError
 from pyinjective.proto.cosmos.auth.v1beta1 import query_pb2 as auth_query, query_pb2_grpc as auth_query_grpc
@@ -3264,8 +3265,7 @@ async def _initialize_tokens_and_markets(self):
         spot_markets = dict()
         derivative_markets = dict()
         binary_option_markets = dict()
-        tokens_by_symbol = dict()
-        tokens_by_denom = dict()
+        tokens_by_symbol, tokens_by_denom = await self._tokens_from_official_lists(network=self.network)
         markets_info = (await self.fetch_spot_markets(market_statuses=["active"]))["markets"]
         valid_markets = (
             market_info
@@ -3400,6 +3400,29 @@ def _token_representation(
 
         return tokens_by_denom[denom]
 
+    async def _tokens_from_official_lists(
+        self,
+        network: Network,
+    ) -> Tuple[Dict[str, Token], Dict[str, Token]]:
+        tokens_by_symbol = dict()
+        tokens_by_denom = dict()
+
+        loader = TokensFileLoader()
+        tokens = await loader.load_tokens(network.official_tokens_list_url)
+
+        for token in tokens:
+            if token.denom is not None and token.denom != "" and token.denom not in tokens_by_denom:
+                unique_symbol = token.symbol
+                for symbol_candidate in [token.symbol, token.name]:
+                    if symbol_candidate not in tokens_by_symbol:
+                        unique_symbol = symbol_candidate
+                        break
+
+                tokens_by_denom[token.denom] = token
+                tokens_by_symbol[unique_symbol] = token
+
+        return tokens_by_symbol, tokens_by_denom
+
     def _initialize_timeout_height_sync_task(self):
         self._cancel_timeout_height_sync_task()
         self._timeout_height_sync_task = asyncio.get_event_loop().create_task(self._timeout_height_sync_process())

pyinjective/core/network.py

@@ -118,6 +118,7 @@ def __init__(
         chain_cookie_assistant: CookieAssistant,
         exchange_cookie_assistant: CookieAssistant,
         explorer_cookie_assistant: CookieAssistant,
+        official_tokens_list_url: str,
         use_secure_connection: Optional[bool] = None,
         grpc_channel_credentials: Optional[ChannelCredentials] = None,
         grpc_exchange_channel_credentials: Optional[ChannelCredentials] = None,
@@ -144,6 +145,7 @@ def __init__(
         self.chain_cookie_assistant = chain_cookie_assistant
         self.exchange_cookie_assistant = exchange_cookie_assistant
         self.explorer_cookie_assistant = explorer_cookie_assistant
+        self.official_tokens_list_url = official_tokens_list_url
         self.grpc_channel_credentials = grpc_channel_credentials
         self.grpc_exchange_channel_credentials = grpc_exchange_channel_credentials
         self.grpc_explorer_channel_credentials = grpc_explorer_channel_credentials
@@ -164,6 +166,7 @@ def devnet(cls):
             chain_cookie_assistant=DisabledCookieAssistant(),
             exchange_cookie_assistant=DisabledCookieAssistant(),
             explorer_cookie_assistant=DisabledCookieAssistant(),
+            official_tokens_list_url="https://github.com/InjectiveLabs/injective-lists/raw/master/tokens/devnet.json",
         )
 
     @classmethod
@@ -218,6 +221,7 @@ def testnet(cls, node="lb"):
             grpc_exchange_channel_credentials=grpc_exchange_channel_credentials,
             grpc_explorer_channel_credentials=grpc_explorer_channel_credentials,
             chain_stream_channel_credentials=chain_stream_channel_credentials,
+            official_tokens_list_url="https://github.com/InjectiveLabs/injective-lists/raw/master/tokens/testnet.json",
         )
 
     @classmethod
@@ -259,6 +263,7 @@ def mainnet(cls, node="lb"):
             grpc_exchange_channel_credentials=grpc_exchange_channel_credentials,
             grpc_explorer_channel_credentials=grpc_explorer_channel_credentials,
             chain_stream_channel_credentials=chain_stream_channel_credentials,
+            official_tokens_list_url="https://github.com/InjectiveLabs/injective-lists/raw/master/tokens/mainnet.json",
         )
 
     @classmethod
@@ -276,6 +281,7 @@ def local(cls):
             chain_cookie_assistant=DisabledCookieAssistant(),
             exchange_cookie_assistant=DisabledCookieAssistant(),
             explorer_cookie_assistant=DisabledCookieAssistant(),
+            official_tokens_list_url="https://github.com/InjectiveLabs/injective-lists/raw/master/tokens/mainnet.json",
         )
 
     @classmethod
@@ -289,6 +295,7 @@ def custom(
         chain_stream_endpoint,
         chain_id,
         env,
+        official_tokens_list_url: str,
         chain_cookie_assistant: Optional[CookieAssistant] = None,
         exchange_cookie_assistant: Optional[CookieAssistant] = None,
         explorer_cookie_assistant: Optional[CookieAssistant] = None,
@@ -322,6 +329,7 @@ def custom(
             chain_cookie_assistant=chain_assistant,
             exchange_cookie_assistant=exchange_assistant,
             explorer_cookie_assistant=explorer_assistant,
+            official_tokens_list_url=official_tokens_list_url,
             grpc_channel_credentials=grpc_channel_credentials,
             grpc_exchange_channel_credentials=grpc_exchange_channel_credentials,
             grpc_explorer_channel_credentials=grpc_explorer_channel_credentials,
@@ -351,6 +359,7 @@ def custom_chain_and_public_indexer_mainnet(
             chain_cookie_assistant=chain_cookie_assistant or DisabledCookieAssistant(),
             exchange_cookie_assistant=mainnet_network.exchange_cookie_assistant,
             explorer_cookie_assistant=mainnet_network.explorer_cookie_assistant,
+            official_tokens_list_url=mainnet_network.official_tokens_list_url,
             grpc_channel_credentials=None,
             grpc_exchange_channel_credentials=mainnet_network.grpc_exchange_channel_credentials,
             grpc_explorer_channel_credentials=mainnet_network.grpc_explorer_channel_credentials,

pyinjective/core/tokens_file_loader.py

@@ -0,0 +1,40 @@
+from typing import Dict, List
+
+import aiohttp
+
+from pyinjective.core.token import Token
+from pyinjective.utils.logger import LoggerProvider
+
+
+class TokensFileLoader:
+    def load_json(self, json: List[Dict]) -> List[Token]:
+        loaded_tokens = []
+
+        for token_info in json:
+            token = Token(
+                name=token_info["name"],
+                symbol=token_info["symbol"],
+                denom=token_info["denom"],
+                address=token_info["address"],
+                decimals=token_info["decimals"],
+                logo=token_info["logo"],
+                updated=-1,
+            )
+
+            loaded_tokens.append(token)
+
+        return loaded_tokens
+
+    async def load_tokens(self, tokens_file_url: str) -> List[Token]:
+        tokens_list = []
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(tokens_file_url) as response:
+                    if response.ok:
+                        tokens_list = await response.json(content_type=None)
+        except Exception as e:
+            LoggerProvider().logger_for_class(logging_class=self.__class__).warning(
+                f"there was an error fetching the list of official tokens: {e}"
+            )
+
+        return self.load_json(tokens_list)

pyproject.toml

@@ -25,7 +25,6 @@ python = "^3.9"
 aiohttp = ">=3.9.2" # Version dependency due to https://github.com/InjectiveLabs/sdk-python/security/dependabot/18
 bech32 = "*"
 bip32 = "*"
-coincurve = "*"
 ecdsa = "*"
 eip712 = "*"
 grpcio = "*"
@@ -35,7 +34,6 @@ mnemonic = "*"
 protobuf = "*"
 requests = "*"
 safe-pysha3 = "*"
-urllib3 = "*"
 websockets = "*"
 web3 = "^6.0"
 
@@ -45,6 +43,7 @@ pytest-asyncio = "*"
 pytest-grpc = "*"
 requests-mock = "*"
 pytest-cov = "^4.1.0"
+pytest-aioresponses = "^0.2.0"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = "^3.4.0"

tests/core/test_network_deprecation_warnings.py

@@ -16,6 +16,7 @@ def test_use_secure_connection_parameter_deprecation_warning(self):
                 chain_id="chain_id",
                 fee_denom="fee_denom",
                 env="env",
+                official_tokens_list_url="https://tokens.url",
                 chain_cookie_assistant=DisabledCookieAssistant(),
                 exchange_cookie_assistant=DisabledCookieAssistant(),
                 explorer_cookie_assistant=DisabledCookieAssistant(),
@@ -40,6 +41,7 @@ def test_use_secure_connection_parameter_in_custom_network_deprecation_warning(s
                 chain_stream_endpoint="chain_stream_endpoint",
                 chain_id="chain_id",
                 env="env",
+                official_tokens_list_url="https://tokens.url",
                 chain_cookie_assistant=DisabledCookieAssistant(),
                 exchange_cookie_assistant=DisabledCookieAssistant(),
                 explorer_cookie_assistant=DisabledCookieAssistant(),

tests/core/test_tokens_file_loader.py

@@ -0,0 +1,112 @@
+import pytest
+
+from pyinjective.core.tokens_file_loader import TokensFileLoader
+
+
+class TestTokensFileLoader:
+    def test_load_tokens(self):
+        tokens_list = [
+            {
+                "address": "",
+                "isNative": True,
+                "decimals": 9,
+                "symbol": "SOL",
+                "name": "Solana",
+                "logo": "https://imagedelivery.net/DYKOWp0iCc0sIkF-2e4dNw/2aa4deed-fa31-4d1a-ba0a-d698b84f3800/public",
+                "coinGeckoId": "solana",
+                "denom": "",
+                "tokenType": "spl",
+                "tokenVerification": "verified",
+                "externalLogo": "solana.png",
+            },
+            {
+                "address": "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270",
+                "isNative": False,
+                "decimals": 18,
+                "symbol": "WMATIC",
+                "logo": "https://imagedelivery.net/DYKOWp0iCc0sIkF-2e4dNw/0d061e1e-a746-4b19-1399-8187b8bb1700/public",
+                "name": "Wrapped Matic",
+                "coinGeckoId": "wmatic",
+                "denom": "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270",
+                "tokenType": "evm",
+                "tokenVerification": "verified",
+                "externalLogo": "polygon.png",
+            },
+        ]
+
+        loader = TokensFileLoader()
+
+        loaded_tokens = loader.load_json(json=tokens_list)
+
+        assert len(loaded_tokens) == 2
+
+        for token, token_info in zip(loaded_tokens, tokens_list):
+            assert token.name == token_info["name"]
+            assert token.symbol == token_info["symbol"]
+            assert token.denom == token_info["denom"]
+            assert token.address == token_info["address"]
+            assert token.decimals == token_info["decimals"]
+            assert token.logo == token_info["logo"]
+
+    @pytest.mark.asyncio
+    async def test_load_tokens_from_url(self, aioresponses):
+        loader = TokensFileLoader()
+        tokens_list = [
+            {
+                "address": "",
+                "isNative": True,
+                "decimals": 9,
+                "symbol": "SOL",
+                "name": "Solana",
+                "logo": "https://imagedelivery.net/DYKOWp0iCc0sIkF-2e4dNw/2aa4deed-fa31-4d1a-ba0a-d698b84f3800/public",
+                "coinGeckoId": "solana",
+                "denom": "",
+                "tokenType": "spl",
+                "tokenVerification": "verified",
+                "externalLogo": "solana.png",
+            },
+            {
+                "address": "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270",
+                "isNative": False,
+                "decimals": 18,
+                "symbol": "WMATIC",
+                "logo": "https://imagedelivery.net/DYKOWp0iCc0sIkF-2e4dNw/0d061e1e-a746-4b19-1399-8187b8bb1700/public",
+                "name": "Wrapped Matic",
+                "coinGeckoId": "wmatic",
+                "denom": "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270",
+                "tokenType": "evm",
+                "tokenVerification": "verified",
+                "externalLogo": "polygon.png",
+            },
+        ]
+
+        aioresponses.get(
+            "https://github.com/InjectiveLabs/injective-lists/raw/master/tokens/mainnet.json", payload=tokens_list
+        )
+        loaded_tokens = await loader.load_tokens(
+            tokens_file_url="https://github.com/InjectiveLabs/injective-lists/raw/master/tokens/mainnet.json"
+        )
+
+        assert len(loaded_tokens) == 2
+
+        for token, token_info in zip(loaded_tokens, tokens_list):
+            assert token.name == token_info["name"]
+            assert token.symbol == token_info["symbol"]
+            assert token.denom == token_info["denom"]
+            assert token.address == token_info["address"]
+            assert token.decimals == token_info["decimals"]
+            assert token.logo == token_info["logo"]
+
+    @pytest.mark.asyncio
+    async def test_load_tokens_from_url_returns_nothing_when_request_fails(self, aioresponses):
+        loader = TokensFileLoader()
+
+        aioresponses.get(
+            "https://github.com/InjectiveLabs/injective-lists/raw/master/tokens/mainnet.json",
+            status=404,
+        )
+        loaded_tokens = await loader.load_tokens(
+            tokens_file_url="https://github.com/InjectiveLabs/injective-lists/raw/master/tokens/mainnet.json"
+        )
+
+        assert len(loaded_tokens) == 0