chore: enable strict mode (#265)

* chore: enable strict mode

* fix: matching ipfs metadata string

* fix: improve handling of undefined get services

* fix: enhanced in memory snapshotting

* fix: currency initialization

* chore: upgrade indexer version

.github/workflows/subql_multi_deploy_workflow.yaml

@@ -28,8 +28,8 @@ jobs:
   subql_deploy_workflow:
     runs-on: ubuntu-latest
     env:
-      SUBQL_CFG_INDEXER_VERSION: "v5.2.5"
-      SUBQL_ETH_INDEXER_VERSION: "v5.1.3"
+      SUBQL_CFG_INDEXER_VERSION: "v5.2.9"
+      SUBQL_ETH_INDEXER_VERSION: "v5.1.7"
       CHAIN_ID: ${{ inputs.chainId }}
       SUBQL_ACCESS_TOKEN: ${{ secrets.accessToken }}
       SUBQL_PROJ_ORG: ${{ inputs.projOrg }}

smoke-tests/timekeeper.test.ts

@@ -12,7 +12,7 @@ describe('SubQl Nodes', () => {
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const response = await subql<any>(`
       {
-        timekeeper(id: "${chainIds[chain]}") {
+        timekeeper(id: "${chainIds[chain as keyof typeof chainIds]}") {
           lastPeriodStart
         }
       }

smoke-tests/tvl.test.ts

@@ -12,6 +12,6 @@ describe('TVL at known intervals', () => {
       { poolSnapshots(filter: { periodStart: { equalTo: "${sampleDate}" } }) { aggregates { sum { normalizedNAV } } } }
     `)
     const { normalizedNAV } = response.data.poolSnapshots.aggregates.sum
-    expect(normalizedNAV).toBe(knownTVL[sampleDate])
+    expect(normalizedNAV).toBe(knownTVL[sampleDate as keyof typeof knownTVL])
   })
 })

src/@types/gobal.d.ts

@@ -1,4 +1,15 @@
-export {}
+import { ApiPromise } from '@polkadot/api'
+import type { Provider } from '@ethersproject/providers'
+import { ApiDecoration } from '@polkadot/api/types'
+import '@subql/types-core/dist/global'
+import { ExtendedCall, ExtendedRpc } from '../helpers/types'
+export type ApiAt = ApiDecoration<'promise'> & {
+  rpc: ApiPromise['rpc'] & ExtendedRpc
+  call: ApiPromise['call'] & ExtendedCall
+}
 declare global {
-  function getNodeEvmChainId(): Promise<string>
+  const api: ApiAt & Provider
+  const unsafeApi: ApiPromise | undefined
+  function getNodeEvmChainId(): Promise<string | undefined>
 }
+export {}

src/helpers/ipfsFetch.ts

@@ -1,7 +1,8 @@
 import { IPFS_NODE } from '../config'
 
 export const cid = new RegExp(
-  '(Qm[1-9A-HJ-NP-Za-km-z]{44,}|b[A-Za-z2-7]{58,}|B[A-Z2-7]{58,}|z[1-9A-HJ-NP-Za-km-z]{48,}|F[0-9A-F]{50,})$'
+  '(Qm[1-9A-HJ-NP-Za-km-z]{44,}|b[A-Za-z2-7]{58,}|B[A-Z2-7]{58,}|z[1-9A-HJ-NP-Za-km-z]{48,}|F[0-9A-F]{50,})$',
+  'g'
 )
 
 export async function readIpfs<T extends Record<string, unknown>>(ipfsId: string): Promise<T> {

src/helpers/paginatedGetter.ts

@@ -1,8 +1,9 @@
-import type { Entity, FieldsExpression, GetOptions } from '@subql/types-core'
+import type { Entity, FieldsExpression } from '@subql/types-core'
+import { EntityClass, EntityProps } from './stateSnapshot'
 
 export async function paginatedGetter<E extends Entity>(
   entityService: EntityClass<E>,
-  filter: FieldsExpression<E>[]
+  filter: FieldsExpression<EntityProps<E>>[]
 ): Promise<E[]> {
   const results: E[] = []
   const batch = 100
@@ -15,11 +16,5 @@ export async function paginatedGetter<E extends Entity>(
     })
     amount = results.push(...entities)
   } while (entities.length === batch)
-  return results as E[]
-}
-
-export interface EntityClass<E extends Entity> {
-  new (...args): E
-  getByFields(filter: FieldsExpression<E>[], options?: GetOptions<E>): Promise<E[]>
-  create(record): E
+  return results
 }

src/helpers/stateSnapshot.test.ts

@@ -48,15 +48,9 @@ describe('Given a populated pool,', () => {
     set.mockReset()
     getByFields.mockReset()
     getByFields.mockReturnValue([pool])
-    await substrateStateSnapshotter<Pool, PoolSnapshot>(
-      'periodId',
-      periodId,
-      Pool,
-      PoolSnapshot,
-      block,
-      'isActive',
-      true
-    )
+    await substrateStateSnapshotter<Pool, PoolSnapshot>('periodId', periodId, Pool, PoolSnapshot, block, [
+      ['isActive', '=', true],
+    ])
     expect(store.getByFields).toHaveBeenNthCalledWith(
       1,
       'Pool',
@@ -78,8 +72,7 @@ describe('Given a populated pool,', () => {
       Pool,
       PoolSnapshot,
       block,
-      'type',
-      'ALL',
+      [['type', '=', 'ALL']],
       'poolId'
     )
     expect(store.set).toHaveBeenNthCalledWith(

src/helpers/stateSnapshot.ts

@@ -1,5 +1,5 @@
-import { EntityClass, paginatedGetter } from './paginatedGetter'
-import type { Entity } from '@subql/types-core'
+import { paginatedGetter } from './paginatedGetter'
+import type { Entity, FieldsExpression, FunctionPropertyNames, GetOptions } from '@subql/types-core'
 import { EthereumBlock } from '@subql/types-ethereum'
 import { SubstrateBlock } from '@subql/types'
 /**
@@ -16,112 +16,167 @@ import { SubstrateBlock } from '@subql/types'
  * @param resetPeriodStates - (optional) reset properties ending in ByPeriod to 0 after snapshot (defaults to true).
  * @returns A promise resolving when all state manipulations in the DB is completed
  */
-async function stateSnapshotter<T extends SnapshottableEntity, U extends SnapshottedEntityProps>(
-  relationshipField: ForeignKey,
+async function stateSnapshotter<T extends SnapshottableEntity, U extends SnapshottedEntity<T>>(
+  relationshipField: StringForeignKeys<SnapshotAdditions>,
   relationshipId: string,
   stateModel: EntityClass<T>,
   snapshotModel: EntityClass<U>,
   block: { number: number; timestamp: Date },
-  filterKey?: keyof T,
-  filterValue?: T[keyof T],
-  fkReferenceName?: ForeignKey,
+  filters?: FieldsExpression<EntityProps<T>>[],
+  fkReferenceField?: StringForeignKeys<U>,
   resetPeriodStates = true,
-  blockchainId: T['blockchainId'] = '0'
+  blockchainId = '0'
 ): Promise<void[]> {
   const entitySaves: Promise<void>[] = []
   logger.info(`Performing snapshots of ${stateModel.prototype._name} for blockchainId ${blockchainId}`)
-  const filter: Parameters<typeof paginatedGetter<T>>[1] = [['blockchainId', '=', blockchainId]]
-  if (filterKey && filterValue) filter.push([filterKey, '=', filterValue])
-  const stateEntities = (await paginatedGetter(stateModel, filter)) as SnapshottableEntity[]
+  const filter = [['blockchainId', '=', blockchainId]] as FieldsExpression<EntityProps<T>>[]
+  if (filters) filter.push(...filters)
+  const stateEntities = await paginatedGetter(stateModel, filter)
   if (stateEntities.length === 0) logger.info(`No ${stateModel.prototype._name} to snapshot!`)
   for (const stateEntity of stateEntities) {
     const blockNumber = block.number
-    const { id, ...copyStateEntity } = stateEntity
-    logger.info(`Snapshotting ${stateModel.prototype._name}: ${id}`)
-    const snapshotEntity: SnapshottedEntityProps = snapshotModel.create({
-      ...copyStateEntity,
-      id: `${id}-${blockNumber.toString()}`,
+    const snapshot: SnapshottedEntity<T> = {
+      ...stateEntity,
+      id: `${stateEntity.id}-${blockNumber}`,
       timestamp: block.timestamp,
       blockNumber: blockNumber,
       [relationshipField]: relationshipId,
-    })
-    if (fkReferenceName) snapshotEntity[fkReferenceName] = stateEntity.id
-
+    }
+    logger.info(`Snapshotting ${stateModel.prototype._name}: ${stateEntity.id}`)
+    const snapshotEntity = snapshotModel.create(snapshot as U)
+    if (fkReferenceField) snapshotEntity[fkReferenceField] = stateEntity.id as U[StringForeignKeys<U>]
     const propNames = Object.getOwnPropertyNames(stateEntity)
-    const propNamesToReset = propNames.filter((propName) => propName.endsWith('ByPeriod')) as ResettableKey[]
+    const propNamesToReset = propNames.filter((propName) => propName.endsWith('ByPeriod')) as ResettableKeys<T>[]
     if (resetPeriodStates) {
       for (const propName of propNamesToReset) {
-        logger.debug(`resetting ${stateEntity._name.toLowerCase()}.${propName} to 0`)
-        stateEntity[propName] = BigInt(0)
+        logger.debug(`resetting ${stateEntity._name?.toLowerCase()}.${propName} to 0`)
+        stateEntity[propName] = BigInt(0) as T[ResettableKeys<T>]
       }
       entitySaves.push(stateEntity.save())
     }
     entitySaves.push(snapshotEntity.save())
   }
   return Promise.all(entitySaves)
 }
-export function evmStateSnapshotter<T extends SnapshottableEntity, U extends SnapshottedEntityProps>(
-  relationshipField: ForeignKey,
+export function evmStateSnapshotter<T extends SnapshottableEntity, U extends SnapshottedEntity<T>>(
+  relationshipField: StringForeignKeys<SnapshotAdditions>,
   relationshipId: string,
   stateModel: EntityClass<T>,
   snapshotModel: EntityClass<U>,
   block: EthereumBlock,
-  filterKey?: keyof T,
-  filterValue?: T[keyof T],
-  fkReferenceName?: ForeignKey,
+  filters?: FieldsExpression<EntityProps<T>>[],
+  fkReferenceName?: StringForeignKeys<U>,
   resetPeriodStates = true
 ): Promise<void[]> {
   const formattedBlock = { number: block.number, timestamp: new Date(Number(block.timestamp) * 1000) }
-  return stateSnapshotter<T, U>(
+  return stateSnapshotter(
     relationshipField,
     relationshipId,
     stateModel,
     snapshotModel,
     formattedBlock,
-    filterKey,
-    filterValue,
+    filters,
     fkReferenceName,
     resetPeriodStates,
     '1'
   )
 }
 
-export function substrateStateSnapshotter<T extends SnapshottableEntity, U extends SnapshottedEntityProps>(
-  relationshipField: ForeignKey,
+export async function statesSnapshotter<T extends SnapshottableEntity, U extends SnapshottedEntity<T>>(
+  relationshipField: StringForeignKeys<SnapshotAdditions>,
+  relationshipId: string,
+  stateEntities: T[],
+  snapshotModel: EntityClass<U>,
+  blockInfo: BlockInfo,
+  fkReferenceField?: StringForeignKeys<U>,
+  resetPeriodStates = true
+): Promise<void[]> {
+  const entitySaves: Promise<void>[] = []
+  logger.info(`Performing ${snapshotModel.prototype._name}`)
+  if (stateEntities.length === 0) logger.info('Nothing to snapshot!')
+  for (const stateEntity of stateEntities) {
+    const blockNumber = blockInfo.number
+    const snapshot: SnapshottedEntity<T> = {
+      ...stateEntity,
+      id: `${stateEntity.id}-${blockNumber}`,
+      timestamp: blockInfo.timestamp,
+      blockNumber: blockNumber,
+      [relationshipField]: relationshipId,
+    }
+    logger.info(`Creating ${snapshotModel.prototype._name} for: ${stateEntity.id}`)
+    const snapshotEntity = snapshotModel.create(snapshot as U)
+    if (fkReferenceField) snapshotEntity[fkReferenceField] = stateEntity.id as U[StringForeignKeys<U>]
+    const propNames = Object.getOwnPropertyNames(stateEntity)
+    const propNamesToReset = propNames.filter((propName) => propName.endsWith('ByPeriod')) as ResettableKeys<T>[]
+    if (resetPeriodStates) {
+      for (const propName of propNamesToReset) {
+        logger.debug(`resetting ${stateEntity._name?.toLowerCase()}.${propName} to 0`)
+        stateEntity[propName] = BigInt(0) as T[ResettableKeys<T>]
+      }
+      entitySaves.push(stateEntity.save())
+    }
+    entitySaves.push(snapshotEntity.save())
+  }
+  return Promise.all(entitySaves)
+}
+
+export function substrateStateSnapshotter<T extends SnapshottableEntity, U extends SnapshottedEntity<T>>(
+  relationshipField: StringForeignKeys<SnapshotAdditions>,
   relationshipId: string,
   stateModel: EntityClass<T>,
   snapshotModel: EntityClass<U>,
   block: SubstrateBlock,
-  filterKey?: keyof T,
-  filterValue?: T[keyof T],
-  fkReferenceName?: ForeignKey,
+  filters?: FieldsExpression<EntityProps<T>>[],
+  fkReferenceName?: StringForeignKeys<U>,
   resetPeriodStates = true
 ): Promise<void[]> {
+  if (!block.timestamp) throw new Error('Missing block timestamp')
   const formattedBlock = { number: block.block.header.number.toNumber(), timestamp: block.timestamp }
-  return stateSnapshotter<T, U>(
+  return stateSnapshotter(
     relationshipField,
     relationshipId,
     stateModel,
     snapshotModel,
     formattedBlock,
-    filterKey,
-    filterValue,
+    filters,
     fkReferenceName,
     resetPeriodStates,
     '0'
   )
 }
 
-type ResettableKey = `${string}ByPeriod`
-type ForeignKey = `${string}Id`
+type ResettableKeyFormat = `${string}ByPeriod`
+type ForeignKeyFormat = `${string}Id`
+type ResettableKeys<T> = { [K in keyof T]: K extends ResettableKeyFormat ? K : never }[keyof T]
+type ForeignKeys<T> = { [K in keyof T]: K extends ForeignKeyFormat ? K : never }[keyof T]
+type StringFields<T> = { [K in keyof T]: T[K] extends string | undefined ? K : never }[keyof T]
+type StringForeignKeys<T> = NonNullable<ForeignKeys<T> & StringFields<T>>
 
 export interface SnapshottableEntity extends Entity {
+  save(): Promise<void>
   blockchainId: string
 }
 
-export interface SnapshottedEntityProps extends Entity {
+export interface SnapshotAdditions {
+  save(): Promise<void>
+  id: string
   blockNumber: number
   timestamp: Date
   periodId?: string
   epochId?: string
 }
+
+//type Entries<T> = { [K in keyof T]: [K, T[K]] }[keyof T][]
+export type EntityProps<E extends Entity> = Omit<E, NonNullable<FunctionPropertyNames<E>> | '_name'>
+export type SnapshottedEntity<E extends Entity> = SnapshotAdditions & Partial<Omit<{ [K in keyof E]: E[K] }, 'id'>>
+
+export interface EntityClass<T extends Entity> {
+  prototype: { _name: string }
+  getByFields(filter: FieldsExpression<EntityProps<T>>[], options: GetOptions<EntityProps<T>>): Promise<T[]>
+  create(record: EntityProps<T>): T
+}
+
+export interface BlockInfo {
+  number: number
+  timestamp: Date
+}

src/helpers/types.ts

@@ -106,20 +106,26 @@ export interface EpochSolution extends Enum {
   }
 }
 
+export interface TokensStakingCurrency extends Enum {
+  readonly isBlockRewards: boolean
+  readonly type: 'BlockRewards'
+}
+
 export interface TokensCurrencyId extends Enum {
-  isNative: boolean
-  asNative: null
-  isTranche: boolean
-  asTranche: TrancheCurrency | TrancheCurrencyBefore1400
-  isAUSD: boolean
-  asAUSD: null
-  isForeignAsset: boolean
-  asForeignAsset: u32
-  isStaking: boolean
-  asStaking: Enum
-  isLocalAsset: boolean
-  asLocalAsset: u32
-  type: 'Native' | 'Tranche' | 'Ausd' | 'ForeignAsset' | 'Staking' | 'LocalAsset'
+  readonly isNative: boolean
+  readonly asNative: unknown
+  readonly isTranche: boolean
+  readonly asTranche: TrancheCurrency | TrancheCurrencyBefore1400
+  readonly isAusd: boolean
+  readonly asAusd: unknown
+  readonly isForeignAsset: boolean
+  readonly asForeignAsset: u32
+  readonly isStaking: boolean
+  readonly asStaking: TokensStakingCurrency
+  readonly isLocalAsset: boolean
+  readonly asLocalAsset: u32
+  readonly type: 'Native' | 'Tranche' | 'Ausd' | 'ForeignAsset' | 'Staking' | 'LocalAsset'
+  get value(): TrancheCurrency & TrancheCurrencyBefore1400 & u32 & TokensStakingCurrency
 }
 
 export interface TrancheSolution extends Struct {
@@ -497,7 +503,7 @@ export type PoolFeesList = Vec<PoolFeesOfBucket>
 
 export type OracleFedEvent = ITuple<[feeder: DevelopmentRuntimeOriginCaller, key: OracleKey, value: u128]>
 
-export type ExtendedRpc = typeof api.rpc & {
+export type ExtendedRpc = {
   pools: {
     trancheTokenPrice: PromiseRpcResult<
       AugmentedRpc<(poolId: number | string, trancheId: number[]) => Observable<u128>>
@@ -506,7 +512,7 @@ export type ExtendedRpc = typeof api.rpc & {
   }
 }
 
-export type ExtendedCall = typeof api.call & {
+export type ExtendedCall = {
   loansApi: {
     portfolio: AugmentedCall<'promise', (poolId: string) => Observable<Vec<ITuple<[u64, LoanInfoActivePortfolio]>>>>
     expectedCashflows: AugmentedCall<

src/helpers/validation.ts

@@ -0,0 +1,8 @@
+export function assertPropInitialized<T>(
+  obj: T,
+  propertyName: keyof T,
+  type: 'string' | 'number' | 'bigint' | 'boolean' | 'symbol' | 'undefined' | 'object' | 'function'
+) {
+  if (typeof obj[propertyName] !== type) throw new Error(`Property ${propertyName.toString()} not initialized!`)
+  return obj[propertyName]
+}