[Issue-41] Integrate Avail Ledger app

package.json

@@ -89,12 +89,14 @@
     "@polkadot/api-contract": "^10.12.4",
     "@polkadot/networks": "^12.6.2",
     "@polkadot/types": "^10.12.4",
-    "@polkadot/types-codec": "10.10.1",
+    "@polkadot/types-codec": "^10.12.4",
+    "@polkadot/types-known": "^10.12.4",
     "@polkadot/util": "^12.6.2",
     "@polkadot/util-crypto": "^12.6.2",
     "@subwallet/chain-list": "0.2.60",
     "@subwallet/keyring": "^0.1.5",
     "@subwallet/ui-keyring": "^0.1.5",
+    "@zondax/ledger-substrate": "^0.42.1",
     "babel-core": "^7.0.0-bridge.0",
     "babel-jest": "^29.3.1",
     "browserify-sign": "^4.2.2",

packages/extension-base/package.json

@@ -44,6 +44,7 @@
     "@polkadot/rpc-provider": "^10.12.4",
     "@polkadot/types": "^10.12.4",
     "@polkadot/types-augment": "^10.12.4",
+    "@polkadot/types-known": "^10.12.4",
     "@polkadot/ui-settings": "^3.6.5",
     "@polkadot/util": "^12.6.2",
     "@polkadot/util-crypto": "^12.6.2",

packages/extension-base/src/background/KoniTypes.ts

@@ -22,6 +22,7 @@ import Web3 from 'web3';
 import { RequestArguments, TransactionConfig } from 'web3-core';
 import { JsonRpcPayload, JsonRpcResponse } from 'web3-core-helpers';
 
+import { ExtDef } from '@polkadot/types/extrinsic/signedExtensions/types';
 import { SignerResult } from '@polkadot/types/types/extrinsic';
 import { HexString } from '@polkadot/util/types';
 import { KeypairType } from '@polkadot/util-crypto/types';
@@ -283,6 +284,8 @@ export interface MetadataItem {
   genesisHash: string;
   specVersion: string;
   hexValue: HexString;
+  types: Record<string, Record<string, string> | string>;
+  userExtensions?: ExtDef;
 }
 
 export interface CrowdloanItem {
@@ -2133,6 +2136,8 @@ export interface RequestFindRawMetadata {
 export interface ResponseFindRawMetadata {
   rawMetadata: string;
   specVersion: number;
+  types: Record<string, Record<string, string> | string>;
+  userExtensions?: ExtDef;
 }
 
 export interface ResolveDomainRequest {

packages/extension-base/src/koni/background/handlers/Extension.ts

@@ -42,6 +42,7 @@ import { SwapPair, SwapQuoteResponse, SwapRequest, SwapRequestResult, SwapSubmit
 import { BN_ZERO, convertSubjectInfoToAddresses, createTransactionFromRLP, isSameAddress, reformatAddress, signatureToHex, Transaction as QrTransaction, uniqueStringArray } from '@subwallet/extension-base/utils';
 import { parseContractInput, parseEvmRlp } from '@subwallet/extension-base/utils/eth/parseTransaction';
 import { balanceFormatter, formatNumber } from '@subwallet/extension-base/utils/number';
+import { metadataExpand } from '@subwallet/extension-chains';
 import { MetadataDef } from '@subwallet/extension-inject/types';
 import { createPair } from '@subwallet/keyring';
 import { KeyringPair, KeyringPair$Json, KeyringPair$Meta } from '@subwallet/keyring/types';
@@ -57,6 +58,7 @@ import { TransactionConfig } from 'web3-core';
 
 import { SubmittableExtrinsic } from '@polkadot/api/types';
 import { TypeRegistry } from '@polkadot/types';
+import { Registry } from '@polkadot/types/types';
 import { assert, BN, hexStripPrefix, hexToU8a, isAscii, isHex, u8aToHex, u8aToString } from '@polkadot/util';
 import { addressToEvm, base64Decode, decodeAddress, isAddress, isEthereumAddress, jsonDecrypt, keyExtractSuri, mnemonicGenerate, mnemonicValidate } from '@polkadot/util-crypto';
 import { EncryptedJson, KeypairType, Prefix } from '@polkadot/util-crypto/types';
@@ -3149,7 +3151,7 @@ export default class KoniExtension {
 
     const { payload } = request;
 
-    let registry = new TypeRegistry();
+    let registry: Registry;
 
     let isEvm = false;
 
@@ -3158,14 +3160,19 @@ export default class KoniExtension {
        *  Get the metadata for the genesisHash
        *  @todo: need to handle case metadata store in db
       */
-      const currentMetadata = this.#koniState.knownMetadata.find((meta: MetadataDef) =>
+      const metadata = this.#koniState.knownMetadata.find((meta: MetadataDef) =>
         meta.genesisHash === payload.genesisHash);
 
-      // set the registry before calling the sign function
-      registry.setSignedExtensions(payload.signedExtensions, currentMetadata?.userExtensions);
+      if (metadata) {
+        // we have metadata, expand it and extract the info/registry
+        const expanded = metadataExpand(metadata, false);
 
-      if (currentMetadata) {
-        registry.register(currentMetadata?.types);
+        registry = expanded.registry;
+        registry.setSignedExtensions(payload.signedExtensions, expanded.definition.userExtensions);
+      } else {
+        // we have no metadata, create a new registry
+        registry = new TypeRegistry();
+        registry.setSignedExtensions(payload.signedExtensions);
       }
 
       const [, chainInfo] = this.#koniState.findNetworkKeyByGenesisHash(payload.genesisHash);
@@ -3201,6 +3208,9 @@ export default class KoniExtension {
       if (chainInfo) {
         isEvm = _isChainEvmCompatible(chainInfo);
       }
+    } else {
+      // for non-payload, just create a registry to use
+      registry = new TypeRegistry();
     }
 
     const result = request.sign(registry as unknown as TypeRegistry, pair);
@@ -3949,11 +3959,13 @@ export default class KoniExtension {
   /// Metadata
 
   private async findRawMetadata ({ genesisHash }: RequestFindRawMetadata): Promise<ResponseFindRawMetadata> {
-    const { metadata, specVersion } = await this.#koniState.findMetadata(genesisHash);
+    const { metadata, specVersion, types, userExtensions } = await this.#koniState.findMetadata(genesisHash);
 
     return {
       rawMetadata: metadata,
-      specVersion
+      specVersion,
+      types,
+      userExtensions
     };
   }
 

packages/extension-base/src/koni/background/handlers/State.ts

@@ -2067,7 +2067,9 @@ export default class KoniState {
 
     return {
       metadata: metadata?.hexValue || '',
-      specVersion: parseInt(metadata?.specVersion || '0')
+      specVersion: parseInt(metadata?.specVersion || '0'),
+      types: metadata?.types || {},
+      userExtensions: metadata?.userExtensions
     };
   }
 

packages/extension-base/src/services/chain-service/handler/SubstrateApi.ts

@@ -18,13 +18,24 @@ import { BehaviorSubject } from 'rxjs';
 import { ApiPromise, WsProvider } from '@polkadot/api';
 import { SubmittableExtrinsicFunction } from '@polkadot/api/promise/types';
 import { ApiOptions } from '@polkadot/api/types';
-import { typesBundle } from '@polkadot/apps-config/api';
+import { typesBundle as _typesBundle } from '@polkadot/apps-config/api';
 import { ProviderInterface } from '@polkadot/rpc-provider/types';
 import { TypeRegistry } from '@polkadot/types/create';
-import { Registry } from '@polkadot/types/types';
+import { OverrideBundleDefinition, Registry } from '@polkadot/types/types';
 import { BN, formatBalance } from '@polkadot/util';
 import { defaults as addressDefaults } from '@polkadot/util-crypto/address/defaults';
 
+const typesBundle = { ..._typesBundle };
+
+// Override avail spec for signedExtensions
+const _availSpec: OverrideBundleDefinition = {
+  signedExtensions: availSpec.signedExtensions
+};
+
+if (typesBundle.spec) {
+  typesBundle.spec.avail = _availSpec;
+}
+
 export class SubstrateApi implements _SubstrateApi {
   chainSlug: string;
   api: ApiPromise;
@@ -89,7 +100,7 @@ export class SubstrateApi implements _SubstrateApi {
     const apiOption: ApiOptions = {
       provider,
       typesBundle,
-      registry: this.registry,
+      registry: this.registry, // This line makes this object registry to be the same as the api registry
       noInitWarn: true
     };
 
@@ -108,29 +119,26 @@ export class SubstrateApi implements _SubstrateApi {
     if (externalApiPromise) {
       api = externalApiPromise;
     } else if (_API_OPTIONS_CHAIN_GROUP.acala.includes(this.chainSlug)) {
-      api = new ApiPromise(acalaOptions({ provider, noInitWarn: true }));
+      api = new ApiPromise(acalaOptions(apiOption));
     } else if (_API_OPTIONS_CHAIN_GROUP.turing.includes(this.chainSlug)) {
       api = new ApiPromise({
-        provider,
+        ...apiOption,
         rpc: oakRpc,
-        types: oakTypes,
-        noInitWarn: true
+        types: oakTypes
       });
     } else if (_API_OPTIONS_CHAIN_GROUP.avail.includes(this.chainSlug)) {
       api = new ApiPromise({
-        provider,
+        ...apiOption,
         rpc: availSpec.rpc,
         types: availSpec.types,
-        signedExtensions: availSpec.signedExtensions,
-        noInitWarn: true
+        signedExtensions: availSpec.signedExtensions
       });
     } else if (_API_OPTIONS_CHAIN_GROUP.goldberg.includes(this.chainSlug)) {
       api = new ApiPromise({
-        provider,
+        ...apiOption,
         rpc: goldbergRpc,
         types: goldbergTypes,
-        signedExtensions: availSpec.signedExtensions,
-        noInitWarn: true
+        signedExtensions: availSpec.signedExtensions
       });
     } else {
       api = new ApiPromise(apiOption);
@@ -272,7 +280,11 @@ export class SubstrateApi implements _SubstrateApi {
     this.systemName = systemName.toString();
     this.systemVersion = systemVersion.toString();
 
-    const properties = registry.createType('ChainProperties', { ss58Format: api.registry.chainSS58, tokenDecimals: api.registry.chainDecimals, tokenSymbol: api.registry.chainTokens });
+    const properties = registry.createType('ChainProperties', {
+      ss58Format: api.registry.chainSS58,
+      tokenDecimals: api.registry.chainDecimals,
+      tokenSymbol: api.registry.chainTokens
+    });
     const ss58Format = properties.ss58Format.unwrapOr(DEFAULT_SS58).toNumber();
     const tokenSymbol = properties.tokenSymbol.unwrapOr([formatBalance.getDefaults().unit, ...DEFAULT_AUX]);
     const tokenDecimals = properties.tokenDecimals.unwrapOr([DEFAULT_DECIMALS]);

packages/extension-base/src/services/chain-service/handler/SubstrateChainHandler.ts

@@ -10,6 +10,7 @@ import { _ApiOptions, _SubstrateChainSpec } from '@subwallet/extension-base/serv
 import { _SmartContractTokenInfo, _SubstrateApi } from '@subwallet/extension-base/services/chain-service/types';
 
 import { ContractPromise } from '@polkadot/api-contract';
+import { getSpecExtensions, getSpecTypes } from '@polkadot/types-known';
 import { BN } from '@polkadot/util';
 import { logger as createLogger } from '@polkadot/util/logger';
 import { Logger } from '@polkadot/util/types';
@@ -203,6 +204,31 @@ export class SubstrateChainHandler extends AbstractChainHandler {
 
   public async initApi (chainSlug: string, apiUrl: string, { externalApiPromise, onUpdateStatus, providerName }: Omit<_ApiOptions, 'metadata'> = {}): Promise<SubstrateApi> {
     const existed = this.substrateApiMap[chainSlug];
+    const metadata = await this.parent?.getMetadata(chainSlug);
+
+    const updateMetadata = (substrateApi: SubstrateApi) => {
+      // Update metadata to database with async methods
+      substrateApi.api.isReady.then(async (api) => {
+        const currentSpecVersion = api.runtimeVersion.specVersion.toString();
+        const genesisHash = api.genesisHash.toHex();
+
+        // Avoid date existed metadata
+        if (metadata && metadata.specVersion === currentSpecVersion && metadata.genesisHash === genesisHash) {
+          return;
+        }
+
+        const systemChain = await api.rpc.system.chain();
+
+        this.parent?.upsertMetadata(chainSlug, {
+          chain: chainSlug,
+          genesisHash: genesisHash,
+          specVersion: currentSpecVersion,
+          hexValue: api.runtimeMetadata.asCallsOnly.toHex(), // Shorten metadata to save space
+          types: getSpecTypes(api.registry, systemChain, api.runtimeVersion.specName, api.runtimeVersion.specVersion) as unknown as Record<string, string>,
+          userExtensions: getSpecExtensions(api.registry, systemChain, api.runtimeVersion.specName)
+        }).catch(console.error);
+      }).catch(console.error);
+    };
 
     // Return existed to avoid re-init metadata
     if (existed) {
@@ -212,29 +238,18 @@ export class SubstrateChainHandler extends AbstractChainHandler {
         await existed.updateApiUrl(apiUrl);
       }
 
+      // Update data in case of existed api (if needed - old provider cannot connect)
+      updateMetadata(existed);
+
       return existed;
     }
 
-    const metadata = await this.parent?.getMetadata(chainSlug);
     const apiObject = new SubstrateApi(chainSlug, apiUrl, { providerName, metadata, externalApiPromise });
 
     apiObject.connectionStatusSubject.subscribe(this.handleConnection.bind(this, chainSlug));
     onUpdateStatus && apiObject.connectionStatusSubject.subscribe(onUpdateStatus);
 
-    // Update metadata to database with async methods
-    apiObject.isReady.then((api) => {
-      // Avoid date existed metadata
-      if (metadata && metadata.specVersion === api.specVersion && metadata.genesisHash === api.api.genesisHash.toHex()) {
-        return;
-      }
-
-      this.parent?.upsertMetadata(chainSlug, {
-        chain: chainSlug,
-        genesisHash: api.api.genesisHash.toHex(),
-        specVersion: api.specVersion,
-        hexValue: api.api.runtimeMetadata.toHex()
-      }).catch(console.error);
-    }).catch(console.error);
+    updateMetadata(apiObject);
 
     return apiObject;
   }

packages/extension-base/src/services/migration-service/scripts/ClearMetadataDatabase.ts

@@ -0,0 +1,13 @@
+// Copyright 2019-2022 @subwallet/extension-koni authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import BaseMigrationJob from '@subwallet/extension-base/services/migration-service/Base';
+
+export default class ClearMetadataDatabase extends BaseMigrationJob {
+  public override async run (): Promise<void> {
+    // Clear all old metadata data
+    await this.state.dbService.stores.metadata.clear();
+
+    return Promise.resolve();
+  }
+}

packages/extension-base/src/services/migration-service/scripts/index.ts

@@ -6,6 +6,7 @@ import MigrateTransactionHistoryBySymbol from '@subwallet/extension-base/service
 
 import BaseMigrationJob from '../Base';
 import MigrateAssetSetting from './databases/MigrateAssetSetting';
+import ClearMetadataDatabase from './ClearMetadataDatabase';
 
 export const EVERYTIME = '__everytime__';
 
@@ -35,5 +36,6 @@ export default <Record<string, typeof BaseMigrationJob>>{
   // [`${EVERYTIME}-1`]: AutoEnableChainsTokens
   '1.1.58-0___AVAIL': EnableAvailTuringChain,
   '1.1.62-01': MigrateAssetSetting,
-  '1.1.62-02': MigrateTransactionHistoryBySymbol
+  '1.1.53-02': MigrateTransactionHistoryBySymbol,
+  '1.1.61-01___AVAIL': ClearMetadataDatabase
 };

packages/extension-base/src/utils/registry.ts

@@ -2,21 +2,21 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import { _ChainInfo } from '@subwallet/chain-list/types';
+import { ResponseFindRawMetadata } from '@subwallet/extension-base/background/KoniTypes';
 import { _getChainNativeTokenBasicInfo } from '@subwallet/extension-base/services/chain-service/utils';
 
 import { Metadata, TypeRegistry } from '@polkadot/types';
 import { ChainProperties } from '@polkadot/types/interfaces';
 import { Registry } from '@polkadot/types/types';
 import { HexString } from '@polkadot/util/types';
 
-export const createRegistry = (chain: _ChainInfo, rawMetadata: HexString): Registry => {
+export const createRegistry = (chain: _ChainInfo, data: ResponseFindRawMetadata): Registry => {
   const registry: Registry = new TypeRegistry();
-  const metadata = new Metadata(registry, rawMetadata);
-
-  registry.setMetadata(metadata);
-
+  const metadata = new Metadata(registry, data.rawMetadata as HexString);
   const tokenInfo = _getChainNativeTokenBasicInfo(chain);
 
+  registry.register(data.types);
+  registry.setMetadata(metadata, undefined, data.userExtensions);
   registry.setChainProperties(registry.createType('ChainProperties', {
     ss58Format: chain.substrateInfo?.addressPrefix || 42,
     tokenDecimals: tokenInfo.decimals,

packages/extension-koni-ui/src/constants/ledger.ts

@@ -5,6 +5,17 @@ import { ChainInfoMap } from '@subwallet/chain-list';
 import { ExtrinsicType, LedgerNetwork } from '@subwallet/extension-base/background/KoniTypes';
 
 export const PredefinedLedgerNetwork: LedgerNetwork[] = [
+  {
+    accountName: 'Avail',
+    appName: 'Avail',
+    networkName: 'Avail network',
+    genesisHash: ChainInfoMap.availTuringTest.substrateInfo?.genesisHash || '0xd3d2f3a3495dc597434a99d7d449ebad6616db45e4e4f178f31cc6fa14378b70',
+    icon: 'substrate',
+    network: 'avail',
+    slug: ChainInfoMap.availTuringTest.slug,
+    isDevMode: true,
+    isEthereum: false
+  },
   {
     accountName: 'Polkadot',
     appName: 'Polkadot',