Merge pull request #114 from GeneralMagicio/feat/import-missed-donations

Added sync with ankr feature

config/example.env

@@ -292,4 +292,9 @@ INVERTER_GRAPHQL_ENDPOINT=
 
 # Funding pot service variables
 DELEGATE_PK_FOR_FUNDING_POT=
-ANKR_API_KEY_FOR_FUNDING_POT=
+ANKR_API_KEY_FOR_FUNDING_POT=
+
+# Sync donations with ankr
+ENABLE_ANKR_SYNC=
+ANKR_RPC_URL=
+ANKR_SYNC_CRONJOB_EXPRESSION=

package.json

@@ -7,6 +7,7 @@
     "@adminjs/design-system": "3.1.5",
     "@adminjs/express": "5.0.1",
     "@adminjs/typeorm": "4.0.0",
+    "@ankr.com/ankr.js": "^0.5.2",
     "@apollo/server": "4.11.0",
     "@apollo/server-plugin-landing-page-graphql-playground": "^4.0.1",
     "@chainvine/sdk": "1.1.10",
@@ -132,6 +133,7 @@
     "test:qfRoundRepository": "NODE_ENV=test mocha ./test/pre-test-scripts.ts  ./src/repositories/qfRoundRepository.test.ts",
     "test:qfRoundHistoryRepository": "NODE_ENV=test mocha ./test/pre-test-scripts.ts  ./src/repositories/qfRoundHistoryRepository.test.ts",
     "test:qfRoundService": "NODE_ENV=test mocha ./test/pre-test-scripts.ts  ./src/services/qfRoundService.test.ts",
+    "test:ankrService": "NODE_ENV=test mocha -t 99999 ./src/services/ankrService.test.ts",
     "test:project": "NODE_ENV=test mocha ./test/pre-test-scripts.ts  ./src/entities/project.test.ts",
     "test:notifyDonationsWithSegment": "NODE_ENV=test mocha ./test/pre-test-scripts.ts  ./src/services/cronJobs/notifyDonationsWithSegment.test.ts",
     "test:checkProjectVerificationStatus": "NODE_ENV=test mocha ./test/pre-test-scripts.ts  ./src/services/cronJobs/checkProjectVerificationStatus.test.ts",

src/constants/ankr.ts

@@ -0,0 +1,9 @@
+import config from '../config';
+
+export const ANKR_FETCH_START_TIMESTAMP =
+  (+config.get('ANKR_FETCH_START_TIMESTAMP') as number) ||
+  Math.floor(Date.now() / 1000);
+
+export const ANKR_RPC_URL: string | undefined = config.get('ANKR_RPC_URL') as
+  | string
+  | undefined;

src/constants/qacc.ts

@@ -1,8 +1,10 @@
 import config from '../config';
 
-export const QACC_DONATION_TOKEN_ADDRESS: string =
+export const QACC_DONATION_TOKEN_ADDRESS: string = (
   (config.get('QACC_DONATION_TOKEN_ADDRESS') as string) ||
-  '0xa2036f0538221a77a3937f1379699f44945018d0'; //https://zkevm.polygonscan.com/token/0xa2036f0538221a77a3937f1379699f44945018d0#readContract
+  //https://zkevm.polygonscan.com/token/0x22B21BedDef74FE62F031D2c5c8F7a9F8a4b304D#readContract
+  '0x22B21BedDef74FE62F031D2c5c8F7a9F8a4b304D'
+).toLowerCase();
 export const QACC_DONATION_TOKEN_SYMBOL =
   (config.get('QACC_DONATION_TOKEN_SYMBOL') as string) || 'MATIC';
 export const QACC_DONATION_TOKEN_NAME =

src/entities/ankrState.ts

@@ -0,0 +1,15 @@
+import { Field, ObjectType } from 'type-graphql';
+import { Column, Entity, BaseEntity, PrimaryColumn, Check } from 'typeorm';
+
+@Entity()
+@ObjectType()
+@Check('"id"')
+export class AnkrState extends BaseEntity {
+  @Field(_type => Boolean)
+  @PrimaryColumn()
+  id: boolean;
+
+  @Field()
+  @Column({ type: 'integer' })
+  timestamp: number;
+}

src/entities/donation.ts

@@ -23,6 +23,7 @@ export const DONATION_STATUS = {
 export const DONATION_ORIGINS = {
   IDRISS_TWITTER: 'Idriss',
   DRAFT_DONATION_MATCHING: 'DraftDonationMatching',
+  CHAIN: 'Chain',
   SUPER_FLUID: 'SuperFluid',
 };
 

src/entities/entities.ts

@@ -35,6 +35,7 @@ import { UserEmailVerification } from './userEmailVerification';
 import { EarlyAccessRound } from './earlyAccessRound';
 import { ProjectRoundRecord } from './projectRoundRecord';
 import { ProjectUserRecord } from './projectUserRecord';
+import { AnkrState } from './ankrState';
 
 export const getEntities = (): DataSourceOptions['entities'] => {
   return [
@@ -82,5 +83,7 @@ export const getEntities = (): DataSourceOptions['entities'] => {
     EarlyAccessRound,
     ProjectRoundRecord,
     ProjectUserRecord,
+
+    AnkrState,
   ];
 };

src/repositories/ankrStateRepository.ts

@@ -0,0 +1,20 @@
+import { AnkrState } from '../entities/ankrState';
+
+export const setAnkrTimestamp = async (
+  timestamp: number,
+): Promise<AnkrState> => {
+  let state = await AnkrState.findOne({ where: {} });
+
+  if (!state) {
+    state = AnkrState.create({
+      id: true,
+      timestamp,
+    });
+  } else {
+    state.timestamp = timestamp;
+  }
+  return state.save();
+};
+
+export const getAnkrState = (): Promise<AnkrState | null> =>
+  AnkrState.findOne({ where: {} });

src/server/adminJs/tabs/donationTab.test.ts

@@ -202,7 +202,7 @@ function createDonationTestCases() {
       );
     }
   });
-  it('Should create donations for gnosis safe', async () => {
+  it.skip('Should create donations for gnosis safe', async () => {
     // https://blockscout.com/xdai/mainnet/tx/0x43f82708d1608aa9355c0738659c658b138d54f618e3322e33a4410af48c200b
 
     const tokenPrice = 1;

src/server/bootstrap.ts

@@ -66,6 +66,7 @@ import { Token } from '../entities/token';
 import { ChainType } from '../types/network';
 import { runFetchRoundTokenPrice } from '../services/cronJobs/fetchRoundTokenPrice';
 import { runSyncDataWithInverter } from '../services/cronJobs/syncDataWithInverter';
+import { runSyncWithAnkrTransfers } from '../services/cronJobs/syncWithAnkrTransfers';
 
 Resource.validate = validate;
 
@@ -390,6 +391,10 @@ export async function bootstrap() {
       'initializeCronJobs() after runSyncDataWithInverter() ',
       new Date(),
     );
+
+    if (process.env.ENABLE_ANKR_SYNC === 'true') {
+      runSyncWithAnkrTransfers();
+    }
   }
 
   async function addQAccToken() {

src/services/ankrService.test.ts

@@ -0,0 +1,20 @@
+/* eslint-disable */
+import { fetchAnkrTransfers } from './ankrService';
+
+describe.skip('AnkrService', () => {
+  it('should return the correct value', async () => {
+    const { lastTimeStamp } = await fetchAnkrTransfers({
+      addresses: [
+        '0x1231DEB6f5749EF6cE6943a275A1D3E7486F4EaE',
+        '0x6C6CD8eD08215120949e057f8D60e33842963beF',
+        '0x6E6B4304195FD46c1Cec1F180e5225e7b4351ffF',
+        '0xf081470f5C6FBCCF48cC4e5B82Dd926409DcdD67',
+      ],
+      fromTimestamp: 1730095935,
+      transferHandler: transfer => {
+        console.log(transfer);
+      },
+    });
+    console.log({ lastTimeStamp });
+  });
+});

src/services/ankrService.ts

@@ -0,0 +1,150 @@
+import {
+  AnkrProvider,
+  Blockchain,
+  TokenTransfer,
+  Transaction,
+} from '@ankr.com/ankr.js';
+import { QACC_DONATION_TOKEN_ADDRESS } from '../constants/qacc';
+import { logger } from '../utils/logger';
+import {
+  getAnkrState,
+  setAnkrTimestamp,
+} from '../repositories/ankrStateRepository';
+import { ANKR_FETCH_START_TIMESTAMP, ANKR_RPC_URL } from '../constants/ankr';
+
+function getNetworkIdString(rpcUrl: string): Blockchain {
+  const [, , , networkIdString] = rpcUrl.split('/');
+  return networkIdString as Blockchain;
+}
+function getAdvancedApiEndpoint(rpcUrl) {
+  const dissembled = rpcUrl.split('/');
+  dissembled[3] = 'multichain';
+  const reassembled = dissembled.join('/');
+  return reassembled;
+}
+const pageSize = 10000;
+
+const getAnkrProviderAndNetworkId = ():
+  | {
+      provider: AnkrProvider;
+      networkIdString: Blockchain;
+    }
+  | undefined => {
+  if (!ANKR_RPC_URL) {
+    return undefined;
+  }
+
+  const networkIdString = getNetworkIdString(ANKR_RPC_URL);
+  const provider = new AnkrProvider(getAdvancedApiEndpoint(ANKR_RPC_URL));
+
+  return {
+    provider,
+    networkIdString,
+  };
+};
+
+export const fetchAnkrTransfers = async ({
+  addresses,
+  fromTimestamp,
+  transferHandler,
+}: {
+  addresses: string[];
+  fromTimestamp: number;
+  transferHandler: (transfer: TokenTransfer) => void;
+}): Promise<{ lastTimeStamp: number | undefined }> => {
+  const ankrConfig = getAnkrProviderAndNetworkId();
+  if (!ankrConfig) {
+    logger.error('Ankr provider not configured');
+    return { lastTimeStamp: undefined };
+  }
+  const { provider, networkIdString } = ankrConfig;
+
+  let pageToken: string | undefined = undefined;
+  let lastTimeStamp: number | undefined = undefined;
+  let retries = 0;
+  do {
+    try {
+      const result = await provider.getTokenTransfers({
+        address: addresses,
+        blockchain: networkIdString,
+        fromTimestamp,
+        pageSize,
+        pageToken,
+      });
+
+      retries = 0;
+
+      for (const transfer of result.transfers) {
+        if (
+          transfer.contractAddress?.toLowerCase() ===
+          QACC_DONATION_TOKEN_ADDRESS
+        ) {
+          try {
+            await transferHandler(transfer);
+          } catch (e) {
+            logger.error('Error processing transfer', e);
+
+            // If we fail to process a transfer, we should not update the timestamp
+            return { lastTimeStamp: undefined };
+          }
+        }
+        lastTimeStamp = transfer.timestamp;
+      }
+
+      pageToken = result.nextPageToken;
+    } catch (e) {
+      logger.info('Error fetching transfers', e);
+      if (retries < 10) {
+        retries++;
+        logger.debug('Retrying');
+        continue;
+      } else {
+        throw e;
+      }
+    }
+  } while (pageToken);
+
+  return { lastTimeStamp };
+};
+
+export const processAnkrTransfers = async ({
+  addresses,
+  transferHandler,
+}: {
+  addresses: string[];
+  transferHandler: (transfer: TokenTransfer) => void;
+}): Promise<void> => {
+  const ankrState = await getAnkrState();
+
+  const fromTimestamp = ankrState?.timestamp
+    ? ankrState?.timestamp + 1
+    : ANKR_FETCH_START_TIMESTAMP;
+
+  const { lastTimeStamp } = await fetchAnkrTransfers({
+    addresses,
+    fromTimestamp,
+    transferHandler,
+  });
+
+  if (lastTimeStamp) {
+    await setAnkrTimestamp(lastTimeStamp);
+  }
+};
+
+export const getTransactionByHash = async (
+  hash: string,
+): Promise<Transaction | undefined> => {
+  const ankrConfig = getAnkrProviderAndNetworkId();
+  if (!ankrConfig) {
+    logger.error('Ankr provider not configured');
+    return;
+  }
+  const { provider, networkIdString } = ankrConfig!;
+
+  const response = await provider.getTransactionsByHash({
+    transactionHash: hash,
+    blockchain: networkIdString,
+  });
+
+  return response?.transactions[0];
+};

src/services/chains/evm/draftDonationService.ts

@@ -11,11 +11,11 @@ import { closeTo } from '..';
 import { findTokenByNetworkAndAddress } from '../../../utils/tokenUtils';
 import { ITxInfo } from '../../../types/etherscan';
 import { DONATION_ORIGINS, Donation } from '../../../entities/donation';
-import { DonationResolver } from '../../../resolvers/donationResolver';
 import { ApolloContext } from '../../../types/ApolloContext';
 import { logger } from '../../../utils/logger';
 import { DraftDonationWorker } from '../../../workers/draftDonationMatchWorker';
 import { normalizeAmount } from '../../../utils/utils';
+import { getDonationResolver } from '../../donationService';
 
 export const isAmountWithinTolerance = (
   callData1,
@@ -247,8 +247,6 @@ async function submitMatchedDraftDonation(
     return;
   }
 
-  const donationResolver = new DonationResolver();
-
   const {
     amount,
     networkId,
@@ -263,7 +261,7 @@ async function submitMatchedDraftDonation(
     logger.debug(
       `Creating donation for draftDonation with ID ${draftDonation.id}`,
     );
-    const donationId = await donationResolver.createDonation(
+    const donationId = await getDonationResolver().createDonation(
       amount,
       tx.hash,
       networkId,

src/services/cronJobs/syncWithAnkrTransfers.ts

@@ -0,0 +1,20 @@
+import { schedule } from 'node-cron';
+import config from '../../config';
+import { logger } from '../../utils/logger';
+import { syncDonationsWithAnkr } from '../donationService';
+
+// As etherscan free plan support 5 request per second I think it's better the concurrent jobs should not be
+// more than 5 with free plan https://etherscan.io/apis
+const cronJobTime =
+  (config.get('ANKR_SYNC_CRONJOB_EXPRESSION') as string) || '*/5 * * * *'; // every 5 minutes starting from 4th minute
+
+export const runSyncWithAnkrTransfers = async () => {
+  logger.debug(
+    'runSyncWithAnkrTrancers() has been called, cronJobTime',
+    cronJobTime,
+  );
+  await syncDonationsWithAnkr();
+  schedule(cronJobTime, async () => {
+    await syncDonationsWithAnkr();
+  });
+};