Merge pull request #113 from embrio-tech/main

20240226 Sprint 28

.envrc

@@ -6,4 +6,4 @@ export DB_HOST=localhost
 export DB_PORT=5432
 export SUBQL_DB_USER=$DB_USER
 export SUBQL_DB_PASS=$DB_PASS
-export CHAIN_ID="centrifuge-cfg"
+export CHAIN_ID="demo"

chains-cfg/_root.yaml

@@ -127,3 +127,33 @@ dataSources:
           filter:
             module: ormlTokens
             method: Withdrawn
+        - handler: handleFeeProposed
+          kind: substrate/EventHandler
+          filter:
+            module: poolFees
+            method: Proposed
+        - handler: handleFeeAdded
+          kind: substrate/EventHandler
+          filter:
+            module: poolFees
+            method: Added
+        - handler: handleFeeRemoved
+          kind: substrate/EventHandler
+          filter:
+            module: poolFees
+            method: Removed
+        - handler: handleFeeCharged
+          kind: substrate/EventHandler
+          filter:
+            module: poolFees
+            method: Charged
+        - handler: handleFeeUncharged
+          kind: substrate/EventHandler
+          filter:
+            module: poolFees
+            method: Uncharged
+        - handler: handleFeePaid
+          kind: substrate/EventHandler
+          filter:
+            module: poolFees
+            method: Paid

chains-cfg/demo.yaml

@@ -6,4 +6,4 @@ network:
     file: ./dist/chaintypes.js
 dataSources:
   - kind: substrate/Runtime
-    startBlock: 1534900 #1st October 2023
+    startBlock: 1549700

chains-cfg/development.yaml

@@ -4,3 +4,8 @@ network:
   chainId: '0x27d6bdae3ea8fc7021792f3ccea5ee62fee37641c6f69d6e8530cfb45ef57a64'
   chaintypes:
     file: ./dist/chaintypes.js
+dataSources:
+  - kind: substrate/Runtime
+    startBlock: 6870 # block first pool was created at
+    filter:
+      modulo: 1000

chains-evm/eth/centrifuge.yaml

@@ -11,23 +11,3 @@ dataSources:
     startBlock: 18721030
     options:
       address: '0x78E9e622A57f70F1E0Ec652A4931E4e278e58142'
-  - kind: ethereum/Runtime
-    startBlock: 11063000
-    options:
-      abi: navFeed
-    assets:
-      navFeed:
-        file: './abi/navfeed.abi.json'
-      reserve:
-        file: './abi/reserve.abi.json'
-      shelf:
-        file: './abi/shelf.abi.json'
-      pile:
-        file: './abi/pile.abi.json'
-    mapping:
-      file: './dist/index.js'
-      handlers:
-        - handler: handleEthBlock
-          kind: ethereum/BlockHandler
-          filter:
-            modulo: 3600

chains-tinlake/centrifuge.yaml

@@ -0,0 +1,43 @@
+network:
+  chainId: '1'
+  endpoint: https://mainnet.infura.io/v3/a4ba76cd4be643618572e7467a444e3a
+  dictionary: https://gx.api.subquery.network/sq/subquery/eth-dictionary
+dataSources:
+  - kind: ethereum/Runtime
+    startBlock: 11063000
+    options:
+      abi: navFeed
+    assets:
+      navFeed:
+        file: ./abi/navfeed.abi.json
+      reserve:
+        file: ./abi/reserve.abi.json
+      shelf:
+        file: ./abi/shelf.abi.json
+      pile:
+        file: ./abi/pile.abi.json
+    mapping:
+      file: ./dist/index.js
+      handlers:
+        - handler: handleEthBlock
+          kind: ethereum/BlockHandler
+          filter:
+            modulo: 3600
+specVersion: 1.0.0
+name: centrifuge-pools
+version: 1.0.0
+runner:
+  node:
+    name: '@subql/node-ethereum'
+    version: '3'
+  query:
+    name: '@subql/query'
+    version: '2'
+description: >-
+  SubQuery API powered by EMBRIO.tech to query Centrifuge chain data for
+  analytics insights on how Centrifuge is unlocking economic opportunity for all
+  by connecting people to borrow and lend money transparently and
+  cost-effectively.
+repository: https://github.com/centrifuge/pools-subql
+schema:
+  file: ./schema.graphql

package.json

@@ -4,13 +4,13 @@
   "description": "Subquery for the pools functionality",
   "main": "dist/index.js",
   "scripts": {
-    "postinstall": "wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O ./node_modules/.bin/yq && chmod +x ./node_modules/.bin/yq",
     "build": "subql build",
     "build:dev": "subql build --mode=dev",
+    "dev": "docker compose down -v --remove-orphans && docker compose pull && docker compose up -d",
     "prepack": "rm -rf dist && subql build",
     "test": "jest",
     "codegen": "subql codegen",
-    "projectgen" : "./project-generator.sh",
+    "projectgen": "./project-generator.sh",
     "generate:defs": "ts-node --skip-project node_modules/.bin/polkadot-types-from-defs --package centrifuge-subql/src/api-interfaces --endpoint 'wss://fullnode.development.cntrfg.com' --input ./src/api-interfaces",
     "generate:meta": "ts-node --skip-project node_modules/.bin/polkadot-types-from-chain --endpoint 'wss://fullnode.development.cntrfg.com' --output ./src/api-interfaces --strict",
     "interface-build": "yarn generate:defs && yarn generate:meta && yarn build",
@@ -34,7 +34,7 @@
     "@jest/globals": "^29.2.0",
     "@polkadot/api": "^10",
     "@polkadot/typegen": "^10",
-    "@subql/cli": "^4.2.5",
+    "@subql/cli": "latest",
     "@subql/testing": "latest",
     "@subql/types": "latest",
     "@types/jest": "^29.1.2",
@@ -47,12 +47,11 @@
     "husky": "^7.0.0",
     "jest": "^29.2.0",
     "lint-staged": "^13.2.0",
-    "node-fetch": "2.6.7",
     "prettier": "^3.1.1",
     "prettier-eslint": "^16.2.0",
     "ts-jest": "^29.0.3",
-    "ts-node": "^8.6.2",
-    "typescript": "^4.1.3"
+    "typescript": "^5.2.2",
+    "yaml-merge-cli": "^1.1.5"
   },
   "lint-staged": {
     "src/**/*.{js,ts}": [
@@ -61,9 +60,5 @@
   },
   "exports": {
     "chaintypes": "./src/chaintypes.ts"
-  },
-  "resolutions": {
-    "ipfs-unixfs": "6.0.6",
-    "typescript": "4.9"
   }
 }

project-generator.sh

@@ -8,13 +8,22 @@ projects:
   - project-centrifuge.yaml
 EOF
 
-yq ". *=d load(\"chains-cfg/${CHAIN_ID}.yaml\")" chains-cfg/_root.yaml > project-centrifuge.yaml
+#CFG
+merge-yaml -i ./chains-cfg/_root.yaml ./chains-cfg/${CHAIN_ID}.yaml -o ./project-centrifuge.yaml
 
+#EVM Chains
 subdirs=$(find chains-evm -mindepth 1 -maxdepth 1 -type d)
 for subdir in $subdirs; do
     if [ -e "$subdir/${CHAIN_ID}.yaml" ]; then
       echo "Generating EVM config for ${subdir##*/}"
-      yq ". *=d load(\"$subdir/${CHAIN_ID}.yaml\")" chains-evm/_root.yaml > project-${subdir##*/}.yaml
+      merge-yaml -i ./$subdir/${CHAIN_ID}.yaml ./chains-evm/_root.yaml -o ./project-${subdir##*/}.yaml
       echo "  - project-${subdir##*/}.yaml" >> subquery-multichain.yaml
     fi
-done
+done
+
+#TINLAKE
+if [ -e "chains-tinlake/${CHAIN_ID}.yaml" ]; then
+  echo "Generating Tinlake config for"
+  merge-yaml -i ./chains-tinlake/${CHAIN_ID}.yaml -o ./project-tinlake.yaml
+  echo "  - project-tinlake.yaml" >> subquery-multichain.yaml
+fi

schema.graphql

@@ -447,3 +447,79 @@ type CurrencyBalance @entity {
 type Blockchain @entity {
   id: ID! #EVM chainId
 }
+
+enum PoolFeeStatus {
+  PROPOSED
+  ADDED
+  REMOVED
+}
+
+enum PoolFeeType {
+  Fixed
+  ChargedUpTo
+}
+
+type PoolFee @entity {
+  id: ID! # poolId - feeId
+  feeId: String!
+  type: PoolFeeType!
+  status: PoolFeeStatus!
+  isActive: Boolean! @index
+
+  createdAt: Date
+  createdAtBlockNumber: Int
+  createdAtEpoch: Int
+
+  pool: Pool! @index
+
+  sumChargedAmount: BigInt #Applies to Fixed ONLY
+  sumAccruedAmount: BigInt #Applies toChargedUpTo ONLY
+  sumPaidAmount: BigInt #Applies to All
+  pendingAmount: BigInt #Applies to All
+
+  sumChargedAmountByPeriod: BigInt #Applies to Fixed ONLY
+  sumAccruedAmountByPeriod: BigInt #Applies toChargedUpTo ONLY
+  sumPaidAmountByPeriod: BigInt #Applies to All
+}
+
+type PoolFeeSnapshot @entity {
+  id: ID! # poolId - feeId - blockNumber
+  feeId: String!
+  poolFee: PoolFee! @index
+
+  timestamp: Date!
+  blockNumber: Int!
+  periodStart: Date! @index
+
+  sumChargedAmount: BigInt #Applies to Fixed ONLY
+  sumAccruedAmount: BigInt #Applies toChargedUpTo ONLY
+  sumPaidAmount: BigInt #Applies to All
+  pendingAmount: BigInt #Applies to All
+
+  sumChargedAmountByPeriod: BigInt #Applies to Fixed ONLY
+  sumAccruedAmountByPeriod: BigInt #Applies toChargedUpTo ONLY
+  sumPaidAmountByPeriod: BigInt #Applies to All
+}
+
+enum PoolFeeTransactionType {
+  PROPOSED
+  ADDED
+  REMOVED
+  CHARGED
+  UNCHARGED
+  PAID
+  ACCRUED
+}
+
+type PoolFeeTransaction @entity {
+  id: ID! # extrinsicHash - epochNumber  - transaction type
+  poolFee: PoolFee! @index
+  type: PoolFeeTransactionType!
+
+  timestamp: Date!
+  blockNumber: Int!
+  epochNumber: Int!
+
+  amount: BigInt
+
+}

src/@types/gobal.d.ts

@@ -1,4 +1,4 @@
 export {}
 declare global {
-  function getNodeChainId(): Promise<string>
+  function getNodeEvmChainId(): Promise<string>
 }

src/chaintypes.ts

@@ -13,6 +13,12 @@ const definitions: OverrideBundleDefinition = {
           outstandingPrincipal: 'Balance',
           outstandingInterest: 'Balance',
         },
+        PoolNav: {
+          navAum: 'Balance',
+          navFees: 'Balance',
+          reserve: 'Balance',
+          total: 'Balance',
+        },
       },
     },
   ],
@@ -48,6 +54,23 @@ const definitions: OverrideBundleDefinition = {
         version: 1,
       },
     ],
+    PoolsApi: [
+      {
+        methods: {
+          nav: {
+            description: 'Get active pool NAV',
+            params: [
+              {
+                name: 'pool_id',
+                type: 'u64',
+              },
+            ],
+            type: 'Option<PoolNav>',
+          },
+        },
+        version: 1,
+      },
+    ],
   },
   rpc: {
     pools: {

src/helpers/stateSnapshot.ts

@@ -30,6 +30,7 @@ async function _stateSnapshotter(
   const entitySaves: Promise<void>[] = []
   logger.info(`Performing snapshots of ${stateModel}`)
   const stateEntities = (await paginatedGetter(stateModel, filterKey, filterValue)) as ResettableEntity[]
+  if(stateEntities.length === 0) logger.info(`No ${stateModel} to snapshot!`)
   for (const stateEntity of stateEntities) {
     const blockNumber = block.number
     const { id, ...copyStateEntity } = stateEntity