Dynamic xcm transfers detection

.gitignore

@@ -4,3 +4,4 @@
 __pycache__
 .pytest_cache
 allure-results
+.idea

chains/v17/chains.json

@@ -3892,7 +3892,10 @@
         "addressPrefix": 32,
         "options": [
             "governance-v1"
-        ]
+        ],
+        "additional": {
+            "feeViaRuntimeCall": true
+        }
     },
     {
         "chainId": "7dd99936c1e9e6d1ce7d90eb6f33bea8393b4bf87677d675aa63c9cb3e8c5b5b",

scripts/utils/chain_model.py

@@ -1,17 +1,32 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import List, Union, Callable, TypeVar, Tuple
+
+from scalecodec import ScaleBytes
+
 from .substrate_interface import create_connection_by_url
 from .metadata_interaction import get_properties
 
 from substrateinterface import SubstrateInterface
 
+T = TypeVar('T')
+
 
 class Chain():
-    substrate: SubstrateInterface
+    substrate: SubstrateInterface | None
+
+    assets: List[ChainAsset]
 
-    def __init__(self, arg):
+    _type_registry: dict
+
+    _type_cache: dict = {}
+
+    def __init__(self, arg, type_registry=None):
         self.chainId = arg.get("chainId")
         self.parentId = arg.get("parentId")
         self.name = arg.get("name")
-        self.assets = arg.get("assets")
+        self.assets = [ChainAsset(data, self, self._type_cache) for data in arg.get("assets")]
         self.nodes = arg.get("nodes")
         self.explorers = arg.get("explorers")
         self.addressPrefix = arg.get("addressPrefix")
@@ -20,10 +35,14 @@ def __init__(self, arg):
         self.substrate = None
         self.properties = None
 
+        self._type_registry = type_registry
+
     def create_connection(self) -> SubstrateInterface:
         for node in self.nodes:
             try:
-                self.substrate = create_connection_by_url(node.get('url'))
+                print("Connecting to ", node.get('url'))
+                self.substrate = create_connection_by_url(node.get('url'), type_registry=self._type_registry)
+                print("Connected to ", node.get('url'))
                 return self.substrate
                 # if self.substrate.websocket.connected is True:
                 #     return self.substrate
@@ -34,12 +53,189 @@ def create_connection(self) -> SubstrateInterface:
 
         print("Can't connect to all nodes of network", self.name)
 
+    def recreate_connection(self) -> SubstrateInterface:
+        if self.substrate is None:
+            raise Exception("No connection was created before")
+
+        for node in self.nodes:
+            try:
+                print("Connecting to ", node.get('url'))
+                self.substrate.url = node.get('url')
+                self.substrate.connect_websocket()
+                print("Connected to ", node.get('url'))
+                return self.substrate
+                # if self.substrate.websocket.connected is True:
+                #     return self.substrate
+                # print(f"Can't connect to endpoint {node.get('url')} for the {self.name}")
+            except:
+                print("Can't connect to that node")
+                continue
+
+        print("Can't connect to all nodes of network", self.name)
 
     def close_substrate_connection(self):
         self.substrate.close()
 
-
     def init_properties(self):
         if (self.properties):
-                return self.substrate
+            return self.substrate
         self.properties = get_properties(self.substrate)
+
+    def has_evm_addresses(self):
+        return self.options is not None and "ethereumBased" in self.options
+
+    def get_asset_by_symbol(self, symbol: str) -> ChainAsset:
+        return next((a for a in self.assets if a.symbol == symbol))
+
+    def get_asset_by_id(self, id: int) -> ChainAsset:
+        return next((a for a in self.assets if a.id == id))
+
+    def get_utility_asset(self) -> ChainAsset:
+        return self.get_asset_by_id(0)
+
+    def access_substrate(self, action: Callable[[SubstrateInterface], T]) -> T:
+        if self.substrate is None:
+            self.create_connection()
+
+        try:
+            return action(self.substrate)
+        except Exception as e:
+            print("Attempting to re-create connection after receiving error", e)
+            # try re-connecting socket and performing action once again
+            self.recreate_connection()
+            return action(self.substrate)
+
+
+class ChainAsset:
+    _data: dict
+
+    symbol: str
+    type: AssetType
+    precision: int
+
+    id: int
+
+    chain: Chain
+
+    def __init__(self, data: dict, chain: Chain, chain_cache: dict):
+        self._data = data
+
+        self.id = data["assetId"]
+        self.symbol = data["symbol"]
+        self.type = self._construct_type(data, chain, chain_cache)
+        self.precision = data["precision"]
+        self.chain = chain
+
+    # Backward-compatible override for code that still thinks this is a dict
+    def __getitem__(self, item):
+        return self._data[item]
+
+    def unified_symbol(self) -> str:
+        return self.symbol.removeprefix("xc")
+
+    def planks(self, amount: int | float) -> int:
+        return amount * 10**self.precision
+
+    def full_chain_asset_id(self) -> Tuple[str, int]:
+        return self.chain.chainId, self.id
+
+    @staticmethod
+    def _construct_type(data: dict, chain: Chain, chain_cache: dict) -> AssetType:
+        type_label = data.get("type", "native")
+        type_extras = data.get("typeExtras", {})
+
+        match type_label:
+            case "native":
+                return NativeAssetType()
+            case "statemine":
+                return StatemineAssetType(type_extras)
+            case "orml":
+                return OrmlAssetType(type_extras, chain, chain_cache)
+            case _:
+                return UnsupportedAssetType()
+
+
+class AssetType(ABC):
+
+    @abstractmethod
+    def encodable_asset_id(self) -> object | None:
+        pass
+
+
+class NativeAssetType(AssetType):
+
+    def encodable_asset_id(self):
+        return None
+
+
+class UnsupportedAssetType(AssetType):
+
+    def encodable_asset_id(self) -> object | None:
+        raise Exception("Unsupported")
+
+
+class StatemineAssetType(AssetType):
+    _pallet_name: str
+    _asset_id: str
+
+    def __init__(self, type_extras: dict):
+        self._pallet_name = type_extras.get("palletName", "Assets")
+        self._asset_id = type_extras["assetId"]
+
+    def pallet_name(self) -> str:
+        return self._pallet_name
+
+    def encodable_asset_id(self) -> object | None:
+        if self._asset_id.startswith("0x"):
+            return ScaleBytes(self._asset_id)
+        else:
+            return int(self._asset_id)
+
+
+_orml_pallet_candidates = ["Tokens", "Currencies"]
+_orml_pallet_cache_key = "orml_pallet_name"
+
+
+class OrmlAssetType(AssetType):
+    _asset_id: str
+    _asset_type_scale: str
+
+    _chain: Chain
+
+    _chain_cache: dict
+
+    def __init__(self, type_extras: dict, chain: Chain, chain_cache: dict):
+        self._asset_id = type_extras["currencyIdScale"]
+        self._asset_type_scale = type_extras["currencyIdType"].replace(".", "::")
+        self._chain = chain
+        self._chain_cache = chain_cache
+
+    def encodable_asset_id(self) -> object | None:
+        return self._chain.access_substrate(
+            lambda s: self.__create_encodable_id(s)
+        )
+
+    def __create_encodable_id(self, substrate: SubstrateInterface):
+        encoded_bytes = ScaleBytes(self._asset_id)
+
+        return substrate.create_scale_object(self._asset_type_scale, encoded_bytes).process()
+
+    def pallet_name(self) -> str:
+        if self._chain_cache.get(_orml_pallet_cache_key) is None:
+            self._chain_cache[_orml_pallet_cache_key] = self.__determine_pallet_name()
+
+        return self._chain_cache[_orml_pallet_cache_key]
+
+    def __determine_pallet_name(self) -> str:
+        return self._chain.access_substrate(
+            lambda s: self.__determine_pallet_name_using_connection(s)
+        )
+
+    @staticmethod
+    def __determine_pallet_name_using_connection(substrate: SubstrateInterface) -> str:
+        for candidate in _orml_pallet_candidates:
+            module = substrate.get_metadata_module(candidate)
+            if module is not None:
+                return candidate
+
+        raise Exception(f"Failed to find orml pallet name, searched for {_orml_pallet_candidates}")

scripts/utils/substrate_interface.py

@@ -13,7 +13,7 @@
     "headers": {"Origin": "polkadot.js.org"}
 }
 
-def create_connection_by_url(url, use_remote_preset=False, ws_otpions=default_ws_options) -> SubstrateInterface:
+def create_connection_by_url(url, use_remote_preset=False, ws_otpions=default_ws_options, type_registry=None) -> SubstrateInterface:
     """Returns substrate interface object
 
     Args:
@@ -28,7 +28,8 @@ def create_connection_by_url(url, use_remote_preset=False, ws_otpions=default_ws
     """
     try:
         substrate = SubstrateInterface(
-                url=url, use_remote_preset=use_remote_preset, ws_options=ws_otpions
+                url=url, use_remote_preset=use_remote_preset, ws_options=ws_otpions,
+                type_registry=type_registry
             )
     except Exception as err:
         print(f"⚠️ Can't connect by {url}, check it is available? \n {err}")

scripts/xcm_transfers/construct_dynamic_config_base.py

@@ -0,0 +1,44 @@
+import json
+
+from scripts.utils.work_with_data import get_data_from_file, write_data_to_file
+from scripts.xcm_transfers.utils.xcm_config_files import XCMConfigFiles
+from scripts.xcm_transfers.xcm.registry.xcm_registry_builder import build_polkadot_xcm_registry
+
+config_files = XCMConfigFiles(
+    chains="../../chains/v21/chains_dev.json",
+    xcm_legacy_config="../../xcm/v7/transfers_dev.json",
+    xcm_additional_data="xcm_registry_additional_data.json",
+    xcm_dynamic_config="../../xcm/v7/transfers_dynamic_dev.json",
+)
+
+registry = build_polkadot_xcm_registry(config_files)
+
+config = get_data_from_file(config_files.xcm_legacy_config)
+
+new_config = {"assetsLocation": {}}
+
+for reserve_id, reserve_config in config["assetsLocation"].items():
+    reserve_config.pop("reserveFee", None)
+    new_config["assetsLocation"][reserve_id] = reserve_config
+
+reserve_overrides = registry.reserves.dump_overrides()
+
+# Remove redundant declarations from overrides
+for chain_id, chain_overrides in list(reserve_overrides.items()):
+    chain = registry.get_chain_or_none(chain_id)
+
+    if chain is None:
+        del reserve_overrides[chain_id]
+        continue
+
+    for asset_id, reserve_id in list(chain_overrides.items()):
+        asset = chain.chain.get_asset_by_id(asset_id)
+        if asset.symbol == reserve_id:
+            del chain_overrides[asset_id]
+
+    if len(chain_overrides) == 0:
+        del reserve_overrides[chain_id]
+
+new_config["reserveIdOverrides"] = reserve_overrides
+
+write_data_to_file("../../xcm/v7/transfers_dynamic_dev.json", json.dumps(new_config, indent=2))