From 7ef0064403623080779f078214429a00bca41c6a Mon Sep 17 00:00:00 2001 From: uvorbs Date: Sun, 13 Oct 2024 11:05:55 +0300 Subject: [PATCH 1/3] increase page size --- .gitignore | 6 ++++++ src/ethereum/event-fetcher-lookahead.ts | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f125f40..b59ab4a 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,9 @@ typings/ # version .version + +# code generated +status/ + +# IDE +.vscode/ \ No newline at end of file diff --git a/src/ethereum/event-fetcher-lookahead.ts b/src/ethereum/event-fetcher-lookahead.ts index 599bcdc..9d423b4 100644 --- a/src/ethereum/event-fetcher-lookahead.ts +++ b/src/ethereum/event-fetcher-lookahead.ts @@ -25,8 +25,8 @@ export class LookaheadEventFetcher extends EventFetcher { protected autoscaleStreak = 0; static DefaultAutoscaleOptions: AutoscaleOptions = { - initialPageSize: 100000, - maxPageSize: 1000000, + initialPageSize: 500000, + maxPageSize: 5000000, minPageSize: 1000, pageGrowFactor: 2, pageGrowAfter: 5, From 9186e4f8e9c030e8eb3f2a26a8933b203f664220 Mon Sep 17 00:00:00 2001 From: uvorbs Date: Sun, 13 Oct 2024 11:07:14 +0300 Subject: [PATCH 2/3] fetch blockTime --- src/ethereum/ethereum-reader.ts | 55 +++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/ethereum/ethereum-reader.ts b/src/ethereum/ethereum-reader.ts index 0429045..7142cbe 100644 --- a/src/ethereum/ethereum-reader.ts +++ b/src/ethereum/ethereum-reader.ts @@ -3,9 +3,14 @@ import { Contract, PastEventOptions } from 'web3-eth-contract'; import { toNumber, DailyStats } from '../helpers'; import { EventName, contractByEventName, getAbiForContract } from './types'; import pThrottle from 'p-throttle'; +import fetch from 'node-fetch'; +import https from 'https'; const HTTP_TIMEOUT_SEC = 20; +const subDomain = 'eth-api' +const domain = 'orbs.network' + export type EthereumConfiguration = { EthereumEndpoint: string; EthereumRequestsPerSecondLimit: number; @@ -14,9 +19,16 @@ export type EthereumConfiguration = { export class EthereumReader { private web3: Web3; private throttled?: pThrottle.ThrottledFunction<[], void>; + private agent: https.Agent; + private blockTimeSinceFail: number; + public requestStats = new DailyStats(); constructor(config: EthereumConfiguration) { + this.agent = new https.Agent({ + maxSockets: 5, + }); + this.blockTimeSinceFail = 0; this.web3 = new Web3( new Web3.providers.HttpProvider(config.EthereumEndpoint, { keepAlive: true, @@ -34,7 +46,50 @@ export class EthereumReader { return this.web3.eth.getBlockNumber(); } + // orbs GET api dediated to serve block time from cache + // for faster sync time + + + async getBlockTime(blockNumber: number | 'latest'): Promise { + const url = `https://${subDomain}.${domain}/api/blocktime?block=${blockNumber}` + try { + const res = await fetch(url, { + timeout: HTTP_TIMEOUT_SEC * 1000, + agent: this.agent, + headers:{ + "x-module":"management-service" + } + }); + if (res.status >= 400){ + console.error("getBlockTime fetch Error status", res.status) + return null; + } + const text = await res.text(); + const time = Number(text) + if(isNaN(time)){ + return null; + } + return time; + }catch(err){ + console.error("getBlockTime Error:", err) + return null; + } + } async getRefTime(blockNumber: number | 'latest'): Promise { + + // get from cache first + const shouldTry = (this.blockTimeSinceFail == 0 || this.blockTimeSinceFail > 5) + if (blockNumber !== 'latest' && shouldTry){ + this.blockTimeSinceFail = 0; + const blocktime = await this.getBlockTime(blockNumber) + if(blocktime) + return blocktime + } + console.log('getBlockTime failed', blockNumber) + // count calls web3 provider + this.blockTimeSinceFail++; + + // fallback to web3 if (this.throttled) await this.throttled(); this.requestStats.add(1); const block = await this.web3.eth.getBlock(blockNumber); From e5579f50da5069cb1592f1ff63a8e776dd640900 Mon Sep 17 00:00:00 2001 From: uvorbs Date: Sun, 13 Oct 2024 11:52:22 +0300 Subject: [PATCH 3/3] blockTime type number --- src/ethereum/ethereum-reader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ethereum/ethereum-reader.ts b/src/ethereum/ethereum-reader.ts index 7142cbe..ce3189f 100644 --- a/src/ethereum/ethereum-reader.ts +++ b/src/ethereum/ethereum-reader.ts @@ -50,7 +50,7 @@ export class EthereumReader { // for faster sync time - async getBlockTime(blockNumber: number | 'latest'): Promise { + async getBlockTime(blockNumber: number): Promise { const url = `https://${subDomain}.${domain}/api/blocktime?block=${blockNumber}` try { const res = await fetch(url, {