From b1ce45bc18bd7a33eca0fc79916b0bfb11e6114a Mon Sep 17 00:00:00 2001 From: Tanuj Soni Date: Wed, 1 May 2024 16:16:40 +0530 Subject: [PATCH] Add latest cycle records data cache --- src/API.ts | 2 +- src/Config.ts | 6 ++++ src/Data/Collector.ts | 3 +- src/Data/Cycles.ts | 13 +++++++++ src/cache/cycleRecordsCache.ts | 50 ++++++++++++++++++++++++++++++++++ src/dbstore/cycles.ts | 13 ++------- src/dbstore/types.ts | 11 ++++++++ 7 files changed, 85 insertions(+), 13 deletions(-) create mode 100644 src/cache/cycleRecordsCache.ts create mode 100644 src/dbstore/types.ts diff --git a/src/API.ts b/src/API.ts index 658c7187..89a1c0fa 100644 --- a/src/API.ts +++ b/src/API.ts @@ -339,7 +339,7 @@ export function registerRoutes(server: FastifyInstance MAX_CYCLES_PER_REQUEST) count = MAX_CYCLES_PER_REQUEST - const cycleInfo = await CycleDB.queryLatestCycleRecords(count) + const cycleInfo = await Cycles.getLatestCycleRecords(count) const res = Crypto.sign({ cycleInfo, }) diff --git a/src/Config.ts b/src/Config.ts index ba71185e..b0a7f920 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -51,6 +51,9 @@ export interface Config { MAX_CYCLES_PER_REQUEST: number MAX_BETWEEN_CYCLES_PER_REQUEST: number } + cycleRecordsCache: { + enabled: boolean + } } let config: Config = { @@ -101,6 +104,9 @@ let config: Config = { MAX_CYCLES_PER_REQUEST: 100, MAX_BETWEEN_CYCLES_PER_REQUEST: 100, }, + cycleRecordsCache: { + enabled: false, + } } // Override default config params from config file, env vars, and cli args export async function overrideDefaultConfig(file: string): Promise { diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index 52090b6f..d64e94ff 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -16,7 +16,7 @@ import * as Logger from '../Logger' import { nestedCountersInstance } from '../profiler/nestedCounters' import { profilerInstance } from '../profiler/profiler' import { getCurrentCycleCounter, shardValuesByCycle, computeCycleMarker } from './Cycles' -import { bulkInsertCycles, Cycle as DbCycle, queryCycleByMarker, updateCycle } from '../dbstore/cycles' +import { bulkInsertCycles, queryCycleByMarker, updateCycle } from '../dbstore/cycles' import * as State from '../State' import * as Utils from '../Utils' import { DataType, GossipData, adjacentArchivers, sendDataToAdjacentArchivers, TxData } from './GossipData' @@ -28,6 +28,7 @@ import ShardFunction from '../ShardFunctions' import { ConsensusNodeInfo } from '../NodeList' import { verifyAccountHash } from '../shardeum/calculateAccountHash' import { verifyAppReceiptData } from '../shardeum/verifyAppReceiptData' +import { Cycle as DbCycle } from '../dbstore/types' export let storingAccountData = false const processedReceiptsMap: Map = new Map() diff --git a/src/Data/Cycles.ts b/src/Data/Cycles.ts index f4aa04ce..e2e8c3d7 100644 --- a/src/Data/Cycles.ts +++ b/src/Data/Cycles.ts @@ -6,6 +6,8 @@ import { P2P as P2PTypes, StateManager } from '@shardus/types' import { getJson, postJson } from '../P2P' import { profilerInstance } from '../profiler/profiler' import { nestedCountersInstance } from '../profiler/nestedCounters' +import * as cycleDataCache from '../cache/cycleRecordsCache' + import { clearDataSenders, dataSenders, @@ -25,6 +27,8 @@ import { handleLostArchivers } from '../LostArchivers' import ShardFunctions from '../ShardFunctions' import { RequestDataType, queryFromArchivers } from '../API' import { stringifyReduce } from '../profiler/StringifyReduce' +import { addCyclesToCache } from '../cache/cycleRecordsCache' +import { queryLatestCycleRecords } from '../dbstore/cycles' interface ArchiverCycleResponse { cycleInfo: P2PTypes.CycleCreatorTypes.CycleData[] @@ -92,6 +96,7 @@ export async function processCycles(cycles: P2PTypes.CycleCreatorTypes.CycleData cleanOldOriginalTxsMap(cleanupTimestamp) cleanOldReceiptsMap(cleanupTimestamp) } + await addCyclesToCache(cycles) } finally { if (profilerInstance) profilerInstance.profileSectionEnd('process_cycle', false) } @@ -534,3 +539,11 @@ function updateShardValues(cycle: P2PTypes.CycleCreatorTypes.CycleData): void { shardValuesByCycle.delete(shardValuesByCycle.keys().next().value) } } + +export async function getLatestCycleRecords(count: number): Promise { + if (config.cycleRecordsCache.enabled) { + return await cycleDataCache.getLatestCycleRecords(count) + } + + return await queryLatestCycleRecords(count) +} diff --git a/src/cache/cycleRecordsCache.ts b/src/cache/cycleRecordsCache.ts new file mode 100644 index 00000000..5a11922d --- /dev/null +++ b/src/cache/cycleRecordsCache.ts @@ -0,0 +1,50 @@ +import { P2P } from '@shardus/types' +import { config } from '../Config' +import { queryLatestCycleRecords } from '../dbstore/cycles' + +let cachedCycleRecords: P2P.CycleCreatorTypes.CycleData[] = [] +let lastCacheUpdateFromDBRunning = false + +async function updateCacheFromDB(): Promise { + if (lastCacheUpdateFromDBRunning) { + return + } + + lastCacheUpdateFromDBRunning = true + + try { + cachedCycleRecords = await queryLatestCycleRecords(config.REQUEST_LIMIT.MAX_CYCLES_PER_REQUEST) + } catch (error) { + console.log('Error updating latest cache: ', error) + } finally { + lastCacheUpdateFromDBRunning = false + } +} + +export async function addCyclesToCache(cycles: P2P.CycleCreatorTypes.CycleData[]): Promise { + cycles.sort((a, b) => a.counter - b.counter); + + if (cachedCycleRecords.length === 0) { + await updateCacheFromDB() + } + + for (const cycle of cycles) { + if(cycle.counter < cachedCycleRecords[0].counter) { + continue + } + + cachedCycleRecords.unshift(cycle) + } + + if (cachedCycleRecords.length > config.REQUEST_LIMIT.MAX_CYCLES_PER_REQUEST) { + cachedCycleRecords.splice(config.REQUEST_LIMIT.MAX_CYCLES_PER_REQUEST) + } +} + +export async function getLatestCycleRecords(count: number): Promise { + if (cachedCycleRecords.length === 0) { + await updateCacheFromDB() + } + + return cachedCycleRecords.slice(0, count) +} diff --git a/src/dbstore/cycles.ts b/src/dbstore/cycles.ts index 3d0ee79d..7da5676c 100644 --- a/src/dbstore/cycles.ts +++ b/src/dbstore/cycles.ts @@ -1,19 +1,10 @@ import * as db from './sqlite3storage' import { extractValues, extractValuesFromArray } from './sqlite3storage' -import { P2P, StateManager } from '@shardus/types' +import { P2P } from '@shardus/types' import * as Logger from '../Logger' import { config } from '../Config' import { DeSerializeFromJsonString, SerializeToJsonString } from '../utils/serialization' - -export interface Cycle { - counter: P2P.CycleCreatorTypes.CycleData['counter'] - cycleRecord: P2P.CycleCreatorTypes.CycleData - cycleMarker: StateManager.StateMetaDataTypes.CycleMarker -} - -type DbCycle = Cycle & { - cycleRecord: string -} +import { Cycle, DbCycle } from './types' export async function insertCycle(cycle: Cycle): Promise { try { diff --git a/src/dbstore/types.ts b/src/dbstore/types.ts new file mode 100644 index 00000000..179bc525 --- /dev/null +++ b/src/dbstore/types.ts @@ -0,0 +1,11 @@ +import { P2P, StateManager } from "@shardus/types" + +export interface Cycle { + counter: P2P.CycleCreatorTypes.CycleData['counter'] + cycleRecord: P2P.CycleCreatorTypes.CycleData + cycleMarker: StateManager.StateMetaDataTypes.CycleMarker +} + +export type DbCycle = Cycle & { + cycleRecord: string +} \ No newline at end of file