more batching for multicalls

adapters/vfat/hourly_blocks.csv

@@ -1,2 +1,2 @@
 number,timestamp
-9397663,1726210798
+11234667,1729932385

adapters/vfat/src/index.ts

@@ -5,7 +5,6 @@ import fs from 'fs';
 import { write } from 'fast-csv';
 
 import { BlockData, OutputSchemaRow } from './sdk/pancake/types';
-
 import { UserVote, UserPosition } from './sdk/nile/types';
 
 import {
@@ -24,7 +23,6 @@ import { fetchUserVotes } from './sdk/nile/lensDetails';
 import BigNumber from 'bignumber.js';
 
 const pipeline = promisify(stream.pipeline);
-
 const NILE_ADDRESS = '0xAAAac83751090C6ea42379626435f805DDF54DC8'.toLowerCase();
 
 export const getUserTVLByBlock = async ({
@@ -69,10 +67,10 @@ export const getUserTVLByBlock = async ({
   // Get sickles and their owners
   const sickleAddresses = await getSickles(blockNumber);
   const sickleOwners = await getSickleOwners(
-    sickleAddresses.map((s: any) => s.sickle)
+    sickleAddresses.map((s: any) => s.sickle),
+    BigInt(blockNumber) // Pass blockNumber as the second argument
   );
-
-// remove non sickle addresses from balances
+  // Remove non-sickle addresses from balances
   for (const [user, tokenBalances] of Object.entries(balances)) {
     if (!sickleOwners[user]) {
       delete balances[user];
@@ -84,7 +82,7 @@ export const getUserTVLByBlock = async ({
   for (const [user, tokenBalances] of Object.entries(balances)) {
     const owner = (
       (sickleOwners as Record<string, string>)[user] || user
-    ).toLowerCase(); // Replace sickle address with owner address
+    ).toLowerCase();
     if (!updatedBalances[owner]) {
       updatedBalances[owner] = {};
     }
@@ -183,24 +181,13 @@ export const getUserVotesTVLByBlock = async (
     [userAddress: string]: BigNumber;
   };
 
-  const batchSize = 300;
-  let userVotesResult: any[] = [];
-  for (let i = 0; i < userAddresses.length; i += batchSize) {
-    const batch = userAddresses.slice(i, i + batchSize);
-    userVotesResult = userVotesResult.concat(
-      await Promise.all(
-        batch.map((user) => fetchUserVotes(BigInt(blockNumber), user))
-      )
-    );
-  }
+  const userVotesResult = await fetchUserVotes(BigInt(blockNumber), userAddresses);
 
   for (const userFecthedVotes of userVotesResult) {
-    for (const userVote of userFecthedVotes) {
-      const userAddress = userVote.result.userAddress.toLowerCase();
-      tokenBalanceMap[userAddress] = BigNumber(
-        tokenBalanceMap[userAddress] ?? 0
-      ).plus(userVote.result.amount.toString());
-    }
+    const userAddress = userFecthedVotes.result.userAddress.toLowerCase();
+    tokenBalanceMap[userAddress] = BigNumber(
+      tokenBalanceMap[userAddress] ?? 0
+    ).plus(userFecthedVotes.result.amount.toString());
   }
 
   Object.entries(tokenBalanceMap).forEach(([userAddress, balance]) => {
@@ -220,34 +207,28 @@ const readBlocksFromCSV = async (filePath: string): Promise<BlockData[]> => {
 
   await new Promise<void>((resolve, reject) => {
     fs.createReadStream(filePath)
-      .pipe(csv()) // Specify the separator as '\t' for TSV files
+      .pipe(csv())
       .on('data', (row) => {
         const blockNumber = parseInt(row.number, 10);
         const blockTimestamp = parseInt(row.timestamp, 10);
         if (!isNaN(blockNumber) && blockTimestamp) {
           blocks.push({ blockNumber: blockNumber, blockTimestamp });
         }
       })
-      .on('end', () => {
-        resolve();
-      })
-      .on('error', (err) => {
-        reject(err);
-      });
+      .on('end', resolve)
+      .on('error', reject);
   });
 
   return blocks;
 };
 
 readBlocksFromCSV('hourly_blocks.csv')
-  .then(async (blocks: any[]) => {
-    console.log(blocks);
-    const allCsvRows: any[] = []; // Array to accumulate CSV rows for all blocks
+  .then(async (blocks: BlockData[]) => {
+    const allCsvRows: OutputSchemaRow[] = [];
 
     for (const block of blocks) {
       try {
         const result = await getUserTVLByBlock(block);
-        // Accumulate CSV rows for all blocks
         allCsvRows.push(...result);
       } catch (error) {
         console.error(`An error occurred for block ${block.blockNumber}:`, error);
@@ -262,9 +243,7 @@ readBlocksFromCSV('hourly_blocks.csv')
           console.log('CSV file has been written.');
           resolve;
         })
-        .on('error', (err) => {
-          reject(err);
-        });
+        .on('error', reject);
     });
   })
   .catch((err) => {

adapters/vfat/src/sdk/nile/lensDetails.ts

@@ -2,7 +2,7 @@ import { Abi, Address, MulticallParameters, PublicClient } from "viem";
 import { client } from "./config";
 import veNILEAbi from "./abis/veNILE.json";
 
-export const VE_NILE_ADDRESS = "0xaaaea1fb9f3de3f70e89f37b69ab11b47eb9ce6f"; // veNILE
+export const VE_NILE_ADDRESS = "0xaaaea1fb9f3de3f70e89f37b69ab11b47eb9ce6f";
 
 export interface VoteRequest {
   userAddress: string;
@@ -13,71 +13,73 @@ export interface VoteResponse {
   result: VoteRequest;
 }
 
-// Function to fetch user votes with batching
 export const fetchUserVotes = async (
   blockNumber: bigint,
-  userAddress: string,
+  userAddresses: string[],
 ): Promise<VoteResponse[]> => {
   const publicClient = client;
 
-  const userBalanceCall = await multicall(
+  const balanceCalls = userAddresses.map((userAddress) => ({
+    address: VE_NILE_ADDRESS,
+    name: "balanceOf",
+    params: [userAddress],
+  }));
+
+  const userBalances = await batchMulticall(
     publicClient,
     veNILEAbi as Abi,
-    [
-      {
-        address: VE_NILE_ADDRESS,
-        name: "balanceOf",
-        params: [userAddress],
-      },
-    ],
+    balanceCalls,
     blockNumber,
+    200,
+    2000,
   );
 
-  const userBalance = userBalanceCall[0].result as number;
-
-  if (userBalance === 0) return [];
-
-  const calls = [];
-  for (let i = 0; i < userBalance; i++) {
-    calls.push({
-      address: VE_NILE_ADDRESS,
-      name: "tokenOfOwnerByIndex",
-      params: [userAddress, i],
-    });
-  }
+  const tokenCalls: any = [];
+  userBalances.forEach((balance, index) => {
+    const userAddress = userAddresses[index];
+    const userBalance = balance.result as number;
+
+    if (userBalance > 0) {
+      for (let i = 0; i < userBalance; i++) {
+        tokenCalls.push({
+          address: VE_NILE_ADDRESS,
+          name: "tokenOfOwnerByIndex",
+          params: [userAddress, i],
+        });
+      }
+    }
+  });
 
   const userTokensCalls = await batchMulticall(
     publicClient,
     veNILEAbi as Abi,
-    calls,
+    tokenCalls,
     blockNumber,
     500,
-    200
+    200,
   );
 
-  const detailsCall = userTokensCalls.map((call) => {
-    return {
-      address: VE_NILE_ADDRESS,
-      name: "locked",
-      params: [call.result],
-    };
-  });
+  const detailsCalls = userTokensCalls.map((call) => ({
+    address: VE_NILE_ADDRESS,
+    name: "locked",
+    params: [call.result],
+  }));
 
   const res = (await batchMulticall(
     publicClient,
     veNILEAbi as Abi,
-    detailsCall,
+    detailsCalls,
     blockNumber,
-    500,
-    200
+    200,
+    2000,
   )) as any;
 
-  return res.map((r: any) => {
+  return res.map((r: any, index: any) => {
+    const userAddress = userAddresses[Math.floor(index / tokenCalls.length)];
     return { result: { amount: r.result[0], userAddress } };
   }) as VoteResponse[];
 };
 
-// Batch multicall function with a delay
 async function batchMulticall(
   publicClient: PublicClient,
   abi: Abi,
@@ -101,37 +103,13 @@ async function batchMulticall(
       blockNumber,
     };
 
-    // Send the batch of requests
     const res = await publicClient.multicall(call);
     results.push(...res);
 
-    // Introduce delay before sending the next batch
     if (i + batchSize < calls.length) {
       await new Promise((resolve) => setTimeout(resolve, delay));
     }
   }
 
   return results;
 }
-
-// Regular multicall function
-function multicall(
-  publicClient: PublicClient,
-  abi: Abi,
-  calls: any[],
-  blockNumber: bigint,
-) {
-  const call: MulticallParameters = {
-    contracts: calls.map((call) => {
-      return {
-        address: call.address as Address,
-        abi,
-        functionName: call.name,
-        args: call.params,
-      };
-    }),
-    blockNumber,
-  };
-
-  return publicClient.multicall(call);
-}