From df8006f7f4f7d32299f3cc20e19b8606b9a55fd7 Mon Sep 17 00:00:00 2001 From: Timo Erdelt Date: Wed, 24 Apr 2024 15:53:09 +0200 Subject: [PATCH 01/10] fix: fix int-as-string param in mdw paginated calls --- src/clients/mdw-http-client.service.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/clients/mdw-http-client.service.ts b/src/clients/mdw-http-client.service.ts index 54d6348..d9cf125 100644 --- a/src/clients/mdw-http-client.service.ts +++ b/src/clients/mdw-http-client.service.ts @@ -97,7 +97,9 @@ export class MdwHttpClientService { condition: (data: T) => boolean, next: string, ): Promise { - const result = await this.get>(next); + const result = await this.get>( + next + `&int-as-string=${this.INT_AS_STRING}`, + ); if (result.data.filter(condition).length === 0 && result.next) { return result.data.concat( From d14b78f48032ab67f2c1fd7c2ae5f9ffed80fe45 Mon Sep 17 00:00:00 2001 From: Timo Erdelt Date: Wed, 24 Apr 2024 15:53:55 +0200 Subject: [PATCH 02/10] feat: implemented fetching of fiat price via coinmarketcap api --- .env | 2 + package-lock.json | 14 ++++++ package.json | 1 + .../migration.sql | 2 +- prisma/schema.prisma | 2 +- src/clients/clients.module.ts | 18 +++++++- src/clients/coinmarketcap-client.model.ts | 42 +++++++++++++++++ src/clients/coinmarketcap-client.service.ts | 46 +++++++++++++++++++ src/clients/http.service.ts | 16 +++++++ src/lib/utils.ts | 3 ++ ...o-history-importer-v2.service.spec.ts.snap | 14 +++--- ...uidity-info-history-importer-v2.service.ts | 30 +++++++++--- ...nfo-history-v2-db.service.e2e-spec.ts.snap | 2 +- ...ity-info-history-v2-db.service.e2e-spec.ts | 4 +- .../pair-liquidity-info-history-mock-data.ts | 8 ++-- 15 files changed, 180 insertions(+), 24 deletions(-) rename prisma/migrations/{20240417083339_history_v2 => 20240423120012_history_v2}/migration.sql (97%) create mode 100644 src/clients/coinmarketcap-client.model.ts create mode 100644 src/clients/coinmarketcap-client.service.ts create mode 100644 src/clients/http.service.ts diff --git a/.env b/.env index 6499fc5..7de1f03 100644 --- a/.env +++ b/.env @@ -33,3 +33,5 @@ DOC_TOKEN2=ct_JDp175ruWd7mQggeHewSLS1PFXt9AzThCDaFedxon8mF8xTRF #DOC_PAIR=ct_NNmdSt3Ws4r87pESGKrhGb7VJmC8zpZymXNJKHY8bTLaFttsi #DOC_TOKEN1=ct_2dE7Xd7XCg3cwpKWP18VPDwfhz5Miji9FoKMTZN7TYvGt64Kc #DOC_TOKEN2=ct_7ur9ypT3a4tjxxv5iG6zEQDQhysNtCKr6tyc7PkqhtRmEw6yY + +COIN_MARKET_CAP_API_KEY= diff --git a/package-lock.json b/package-lock.json index e93ebf8..4c0cf81 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@prisma/client": "^5.10.2", "dex-contracts-v2": "github:aeternity/dex-contracts-v2", "dotenv": "^16.4.3", + "limiter": "^2.1.0", "lodash": "^4.17.21", "reflect-metadata": "^0.2.1", "rimraf": "^5.0.5", @@ -6895,6 +6896,11 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/just-performance": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/just-performance/-/just-performance-4.3.0.tgz", + "integrity": "sha512-L7RjvtJsL0QO8xFs5wEoDDzzJwoiowRw6Rn/GnvldlchS2JQr9wFYPiwZcDfrbbujEKqKN0tvENdbjXdYhDp5Q==" + }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -6926,6 +6932,14 @@ "node": ">= 0.8.0" } }, + "node_modules/limiter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/limiter/-/limiter-2.1.0.tgz", + "integrity": "sha512-361TYz6iay6n+9KvUUImqdLuFigK+K79qrUtBsXhJTLdH4rIt/r1y8r1iozwh8KbZNpujbFTSh74mJ7bwbAMOw==", + "dependencies": { + "just-performance": "4.3.0" + } + }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", diff --git a/package.json b/package.json index 5887207..d4bb415 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "@prisma/client": "^5.10.2", "dex-contracts-v2": "github:aeternity/dex-contracts-v2", "dotenv": "^16.4.3", + "limiter": "^2.1.0", "lodash": "^4.17.21", "reflect-metadata": "^0.2.1", "rimraf": "^5.0.5", diff --git a/prisma/migrations/20240417083339_history_v2/migration.sql b/prisma/migrations/20240423120012_history_v2/migration.sql similarity index 97% rename from prisma/migrations/20240417083339_history_v2/migration.sql rename to prisma/migrations/20240423120012_history_v2/migration.sql index 0a44208..23c7bfe 100644 --- a/prisma/migrations/20240417083339_history_v2/migration.sql +++ b/prisma/migrations/20240423120012_history_v2/migration.sql @@ -7,7 +7,7 @@ CREATE TABLE "PairLiquidityInfoHistoryV2" ( "reserve1" DECIMAL(100,0) NOT NULL, "deltaReserve0" DECIMAL(100,0) NOT NULL, "deltaReserve1" DECIMAL(100,0) NOT NULL, - "fiatPrice" DECIMAL(100,0) NOT NULL, + "aeUsdPrice" DECIMAL(100,6) NOT NULL, "height" INTEGER NOT NULL, "microBlockHash" TEXT NOT NULL, "microBlockTime" BIGINT NOT NULL, diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 138e550..f336a5c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -85,7 +85,7 @@ model PairLiquidityInfoHistoryV2 { reserve1 Decimal @db.Decimal(100, 0) deltaReserve0 Decimal @db.Decimal(100, 0) deltaReserve1 Decimal @db.Decimal(100, 0) - fiatPrice Decimal @db.Decimal(100, 0) + aeUsdPrice Decimal @db.Decimal(100, 6) height Int microBlockHash String microBlockTime BigInt diff --git a/src/clients/clients.module.ts b/src/clients/clients.module.ts index 272b235..3f6c219 100644 --- a/src/clients/clients.module.ts +++ b/src/clients/clients.module.ts @@ -1,11 +1,25 @@ import { Module } from '@nestjs/common'; +import { CoinmarketcapClientService } from '@/clients/coinmarketcap-client.service'; +import { HttpService } from '@/clients/http.service'; import { MdwHttpClientService } from '@/clients/mdw-http-client.service'; import { MdwWsClientService } from '@/clients/mdw-ws-client.service'; import { SdkClientService } from '@/clients/sdk-client.service'; @Module({ - providers: [MdwHttpClientService, MdwWsClientService, SdkClientService], - exports: [MdwHttpClientService, MdwWsClientService, SdkClientService], + providers: [ + CoinmarketcapClientService, + HttpService, + MdwHttpClientService, + MdwWsClientService, + SdkClientService, + ], + exports: [ + CoinmarketcapClientService, + HttpService, + MdwHttpClientService, + MdwWsClientService, + SdkClientService, + ], }) export class ClientsModule {} diff --git a/src/clients/coinmarketcap-client.model.ts b/src/clients/coinmarketcap-client.model.ts new file mode 100644 index 0000000..ea5ba29 --- /dev/null +++ b/src/clients/coinmarketcap-client.model.ts @@ -0,0 +1,42 @@ +export type AEQuoteData = { + 1700: { + id: number; + name: string; + symbol: string; + is_active: 0 | 1; + is_fiat: 0 | 1; + quotes: [ + { + timestamp: Date; + quote: { + USD: { + percent_change_1h: number; + percent_change_24h: number; + percent_change_7d: number; + percent_change_30d: number; + price: number; + volume_24h: number; + market_cap: number; + total_supply: number; + circulating_supply: number; + timestamp: Date; + }; + }; + }, + ]; + }; +}; + +export type CoinMarketCapResponse = { + status: CoinMarketCapStatus; + data: T; +}; + +export type CoinMarketCapStatus = { + timestamp: Date; + error_code: number; + error_message: string; + elapsed: number; + credit_count: number; + notice: string; +}; diff --git a/src/clients/coinmarketcap-client.service.ts b/src/clients/coinmarketcap-client.service.ts new file mode 100644 index 0000000..2574a6b --- /dev/null +++ b/src/clients/coinmarketcap-client.service.ts @@ -0,0 +1,46 @@ +import { Injectable } from '@nestjs/common'; +import { RateLimiter } from 'limiter'; + +import { + AEQuoteData, + CoinMarketCapResponse, +} from '@/clients/coinmarketcap-client.model'; +import { HttpService } from '@/clients/http.service'; + +@Injectable() +export class CoinmarketcapClientService { + constructor(private httpService: HttpService) {} + private readonly AE_CURRENCY_ID = 1700; + private readonly COUNT = 1; + private readonly INTERVAL = '24h'; + private readonly AUTH_HEADER = { + 'X-CMC_PRO_API_KEY': process.env.COIN_MARKET_CAP_API_KEY || '', + }; + private readonly CALLS_LIMIT = 28; + private readonly CALL_INTERVAL = 'minute'; + private rateLimiter = new RateLimiter({ + tokensPerInterval: this.CALLS_LIMIT, + interval: this.CALL_INTERVAL, + }); + + async getHistoricalPriceDataThrottled(microBlockTime: number) { + await this.rateLimiter.removeTokens(1); + const timeEnd = this.roundMicroBlockTimeDownTo5MinInterval(microBlockTime); + const url = `https://pro-api.coinmarketcap.com/v3/cryptocurrency/quotes/historical?id=${this.AE_CURRENCY_ID}&interval=${this.INTERVAL}&count=${this.COUNT}&time_end=${timeEnd}`; + return this.get>(url); + } + + private async get(url: string): Promise { + return this.httpService.get(url, new Headers(this.AUTH_HEADER)); + } + + private roundMicroBlockTimeDownTo5MinInterval( + microBlockTime: number, + ): string { + const date = new Date(microBlockTime); + date.setMinutes(Math.floor(date.getMinutes() / 5) * 5); + date.setSeconds(0); + date.setMilliseconds(0); + return date.toISOString(); + } +} diff --git a/src/clients/http.service.ts b/src/clients/http.service.ts new file mode 100644 index 0000000..ad3e6e4 --- /dev/null +++ b/src/clients/http.service.ts @@ -0,0 +1,16 @@ +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class HttpService { + async get(url: string, headers?: Headers): Promise { + return fetch(url, { method: 'GET', headers: headers }).then(async (res) => { + if (res.ok) { + return (await res.json()) as Promise; + } else { + throw new Error( + `GET ${url} failed with status ${res.status}. Response body: ${await res.text()}`, + ); + } + }); + } +} diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 2ab9377..95f376c 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -23,6 +23,9 @@ export const pluralize = (count: number, noun: string, suffix = 's') => const parseEnv = (x) => x && JSON.parse(x); export const presentInvalidTokens = parseEnv(process.env.SHOW_INVALID_TOKENS); +export const numberToDecimal = (number: number): Decimal => + new Decimal(number.toString()); + export const bigIntToDecimal = (bigInt: bigint): Decimal => new Decimal(bigInt.toString()); diff --git a/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer-v2.service.spec.ts.snap b/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer-v2.service.spec.ts.snap index 73dc336..d43b674 100644 --- a/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer-v2.service.spec.ts.snap +++ b/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer-v2.service.spec.ts.snap @@ -7,7 +7,7 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should import liquidit "deltaReserve0": "0", "deltaReserve1": "0", "eventType": "CreatePair", - "fiatPrice": "0", + "aeUsdPrice": "0", "height": 10000, "logIndex": 0, "microBlockHash": "mh_hash0", @@ -24,7 +24,7 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should import liquidit "deltaReserve0": "100", "deltaReserve1": "100", "eventType": "PairMint", - "fiatPrice": "0", + "aeUsdPrice": "0", "height": 10001, "logIndex": 2, "microBlockHash": "mh_hash1", @@ -41,7 +41,7 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should import liquidit "deltaReserve0": "100", "deltaReserve1": "100", "eventType": "Sync", - "fiatPrice": "0", + "aeUsdPrice": "0", "height": 20002, "logIndex": 1, "microBlockHash": "mh_hash2", @@ -58,7 +58,7 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should import liquidit "deltaReserve0": "1", "deltaReserve1": "-1", "eventType": "SwapTokens", - "fiatPrice": "0", + "aeUsdPrice": "0", "height": 30003, "logIndex": 2, "microBlockHash": "mh_hash3", @@ -75,7 +75,7 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should import liquidit "deltaReserve0": "-101", "deltaReserve1": "-99", "eventType": "PairBurn", - "fiatPrice": "0", + "aeUsdPrice": "0", "height": 30003, "logIndex": 2, "microBlockHash": "mh_hash3", @@ -97,7 +97,7 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should skip a log if t "deltaReserve0": "1", "deltaReserve1": "-1", "eventType": "SwapTokens", - "fiatPrice": "0", + "aeUsdPrice": "0", "height": 30003, "logIndex": 2, "microBlockHash": "mh_hash3", @@ -119,7 +119,7 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should catch and inser "deltaReserve0": "1", "deltaReserve1": "-1", "eventType": "SwapTokens", - "fiatPrice": "0", + "aeUsdPrice": "0", "height": 30003, "logIndex": 2, "microBlockHash": "mh_hash3", diff --git a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.ts b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.ts index e1a87cd..b258af7 100644 --- a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.ts +++ b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.ts @@ -2,6 +2,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { PairLiquidityInfoHistoryV2 } from '@prisma/client'; import { orderBy } from 'lodash'; +import { CoinmarketcapClientService } from '@/clients/coinmarketcap-client.service'; import { ContractLog } from '@/clients/mdw-http-client.model'; import { MdwHttpClientService } from '@/clients/mdw-http-client.service'; import { ContractAddress } from '@/clients/sdk-client.model'; @@ -9,7 +10,7 @@ import { SdkClientService } from '@/clients/sdk-client.service'; import { PairDbService, PairWithTokens } from '@/database/pair/pair-db.service'; import { PairLiquidityInfoHistoryV2DbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-v2-db.service'; import { PairLiquidityInfoHistoryV2ErrorDbService } from '@/database/pair-liquidity-info-history-error/pair-liquidity-info-history-v2-error-db.service'; -import { bigIntToDecimal, decimalToBigInt } from '@/lib/utils'; +import { bigIntToDecimal, decimalToBigInt, numberToDecimal } from '@/lib/utils'; export enum EventType { Sync = 'Sync', @@ -35,11 +36,12 @@ type DeltaReserveEvent = { @Injectable() export class PairLiquidityInfoHistoryImporterV2Service { constructor( - private mdwClient: MdwHttpClientService, private pairDb: PairDbService, private pairLiquidityInfoHistoryDb: PairLiquidityInfoHistoryV2DbService, private pairLiquidityInfoHistoryErrorDb: PairLiquidityInfoHistoryV2ErrorDbService, + private mdwClient: MdwHttpClientService, private sdkClient: SdkClientService, + private coinmarketcapClient: CoinmarketcapClientService, ) {} readonly logger = new Logger(PairLiquidityInfoHistoryImporterV2Service.name); @@ -65,7 +67,7 @@ export class PairLiquidityInfoHistoryImporterV2Service { `Syncing liquidity info history for ${pairsWithTokens.length} pairs.`, ); - for (const pairWithTokens of pairsWithTokens) { + for (const pairWithTokens of pairsWithTokens.slice(3, 4)) { try { // If an error occurred for this pair recently, skip pair const error = @@ -164,6 +166,9 @@ export class PairLiquidityInfoHistoryImporterV2Service { current.event.eventType === EventType.Sync && succeeding.event.eventType !== EventType.Sync ) { + const aeUsdPrice = await this.fetchPrice( + parseInt(succeeding.log.block_time), + ); liquidityInfo = { pairId: pairWithTokens.id, eventType: succeeding.event.eventType, @@ -171,7 +176,7 @@ export class PairLiquidityInfoHistoryImporterV2Service { reserve1: bigIntToDecimal(current.event.reserve1), deltaReserve0: bigIntToDecimal(succeeding.event.deltaReserve0), deltaReserve1: bigIntToDecimal(succeeding.event.deltaReserve1), - fiatPrice: bigIntToDecimal(0n), + aeUsdPrice: numberToDecimal(aeUsdPrice), height: parseInt(succeeding.log.height), microBlockHash: succeeding.log.block_hash, microBlockTime: BigInt(succeeding.log.block_time), @@ -185,6 +190,9 @@ export class PairLiquidityInfoHistoryImporterV2Service { await this.pairLiquidityInfoHistoryDb.getLastlySyncedLogByPairId( pairWithTokens.id, ); + const aeUsdPrice = await this.fetchPrice( + parseInt(current.log.block_time), + ); liquidityInfo = { pairId: pairWithTokens.id, eventType: current.event.eventType, @@ -204,7 +212,7 @@ export class PairLiquidityInfoHistoryImporterV2Service { decimalToBigInt(lastSyncedLog.reserve1), ) : bigIntToDecimal(0n), - fiatPrice: bigIntToDecimal(0n), + aeUsdPrice: numberToDecimal(aeUsdPrice), height: parseInt(current.log.height), microBlockHash: current.log.block_hash, microBlockTime: BigInt(current.log.block_time), @@ -286,7 +294,7 @@ export class PairLiquidityInfoHistoryImporterV2Service { reserve1: bigIntToDecimal(0n), deltaReserve0: bigIntToDecimal(0n), deltaReserve1: bigIntToDecimal(0n), - fiatPrice: bigIntToDecimal(0n), + aeUsdPrice: bigIntToDecimal(0n), height: parseInt(microBlock.height), microBlockHash: microBlock.hash, microBlockTime: BigInt(microBlock.time), @@ -345,4 +353,14 @@ export class PairLiquidityInfoHistoryImporterV2Service { return undefined; } } + + private fetchPrice(microBlockTime: number): Promise { + return this.coinmarketcapClient + .getHistoricalPriceDataThrottled(microBlockTime) + .then((res) => res.data['1700'].quotes[0].quote.USD.price); + // .catch((err) => { + // console.log(err); + // return 0; + // }); + } } diff --git a/test/e2e/__snapshots__/pair-liquidity-info-history-v2-db.service.e2e-spec.ts.snap b/test/e2e/__snapshots__/pair-liquidity-info-history-v2-db.service.e2e-spec.ts.snap index 1e1ded3..d718b52 100644 --- a/test/e2e/__snapshots__/pair-liquidity-info-history-v2-db.service.e2e-spec.ts.snap +++ b/test/e2e/__snapshots__/pair-liquidity-info-history-v2-db.service.e2e-spec.ts.snap @@ -5,7 +5,7 @@ exports[`PairLiquidityInfoHistoryV2DbService upsert should correctly upsert an e "deltaReserve0": "-500", "deltaReserve1": "-500", "eventType": "PairBurn", - "fiatPrice": "0", + "aeUsdPrice": "0", "height": 200002, "id": 222, "logIndex": 1, diff --git a/test/e2e/pair-liquidity-info-history-v2-db.service.e2e-spec.ts b/test/e2e/pair-liquidity-info-history-v2-db.service.e2e-spec.ts index 2c81785..0f86da8 100644 --- a/test/e2e/pair-liquidity-info-history-v2-db.service.e2e-spec.ts +++ b/test/e2e/pair-liquidity-info-history-v2-db.service.e2e-spec.ts @@ -60,7 +60,7 @@ describe('PairLiquidityInfoHistoryV2DbService', () => { reserve1: new Decimal(500), deltaReserve0: new Decimal(-500), deltaReserve1: new Decimal(-500), - fiatPrice: new Decimal(0), + aeUsdPrice: new Decimal(0), height: 200002, microBlockHash: historyEntry2.microBlockHash, microBlockTime: 2000000000002n, @@ -83,7 +83,7 @@ describe('PairLiquidityInfoHistoryV2DbService', () => { reserve1: new Decimal(500), deltaReserve0: new Decimal(-500), deltaReserve1: new Decimal(-500), - fiatPrice: new Decimal(0), + aeUsdPrice: new Decimal(0), height: 500005, microBlockHash: 'mh_entry5', microBlockTime: 5000000000005n, diff --git a/test/mock-data/pair-liquidity-info-history-mock-data.ts b/test/mock-data/pair-liquidity-info-history-mock-data.ts index 0524c8b..b61257e 100644 --- a/test/mock-data/pair-liquidity-info-history-mock-data.ts +++ b/test/mock-data/pair-liquidity-info-history-mock-data.ts @@ -70,7 +70,7 @@ export const historyEntry1: PairLiquidityInfoHistoryV2 = { reserve1: new Decimal(1000), deltaReserve0: new Decimal(1000), deltaReserve1: new Decimal(1000), - fiatPrice: new Decimal(0), + aeUsdPrice: new Decimal(0), height: 100001, microBlockHash: 'mh_entry1', microBlockTime: 1000000000001n, @@ -89,7 +89,7 @@ export const historyEntry2: PairLiquidityInfoHistoryV2 = { reserve1: new Decimal(950), deltaReserve0: new Decimal(50), deltaReserve1: new Decimal(-50), - fiatPrice: new Decimal(0), + aeUsdPrice: new Decimal(0), height: 200002, microBlockHash: 'mh_entry2', microBlockTime: 2000000000002n, @@ -108,7 +108,7 @@ export const historyEntry3: PairLiquidityInfoHistoryV2 = { reserve1: new Decimal(1000), deltaReserve0: new Decimal(1000), deltaReserve1: new Decimal(1000), - fiatPrice: new Decimal(0), + aeUsdPrice: new Decimal(0), height: 300003, microBlockHash: 'mh_entry3', microBlockTime: 3000000000003n, @@ -127,7 +127,7 @@ export const historyEntry4: PairLiquidityInfoHistoryV2 = { reserve1: new Decimal(950), deltaReserve0: new Decimal(50), deltaReserve1: new Decimal(-50), - fiatPrice: new Decimal(0), + aeUsdPrice: new Decimal(0), height: 300003, microBlockHash: 'mh_entry3', microBlockTime: 3000000000003n, From 1a9c7838a6d5d2ab3454e9c22870b948a31c90d4 Mon Sep 17 00:00:00 2001 From: Timo Erdelt Date: Wed, 24 Apr 2024 16:04:25 +0200 Subject: [PATCH 03/10] refactor: use HttpService in MdwHttpClientService --- src/clients/coinmarketcap-client.service.ts | 1 + src/clients/mdw-http-client.service.ts | 13 ++++--------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/clients/coinmarketcap-client.service.ts b/src/clients/coinmarketcap-client.service.ts index 2574a6b..32b3476 100644 --- a/src/clients/coinmarketcap-client.service.ts +++ b/src/clients/coinmarketcap-client.service.ts @@ -10,6 +10,7 @@ import { HttpService } from '@/clients/http.service'; @Injectable() export class CoinmarketcapClientService { constructor(private httpService: HttpService) {} + private readonly AE_CURRENCY_ID = 1700; private readonly COUNT = 1; private readonly INTERVAL = '24h'; diff --git a/src/clients/mdw-http-client.service.ts b/src/clients/mdw-http-client.service.ts index d9cf125..68c5c23 100644 --- a/src/clients/mdw-http-client.service.ts +++ b/src/clients/mdw-http-client.service.ts @@ -1,5 +1,6 @@ import { Injectable } from '@nestjs/common'; +import { HttpService } from '@/clients/http.service'; import { AccountBalance, Contract, @@ -19,6 +20,8 @@ import { nonNullable } from '@/lib/utils'; @Injectable() export class MdwHttpClientService { + constructor(private httpService: HttpService) {} + private readonly LIMIT = 100; private readonly DIRECTION = 'forward'; private readonly INT_AS_STRING = true; @@ -81,15 +84,7 @@ export class MdwHttpClientService { private async get(url: string): Promise { const fullUrl = `${NETWORKS[nonNullable(process.env.NETWORK_NAME)].middlewareHttpUrl}${url}`; - return fetch(fullUrl).then(async (res) => { - if (res.ok) { - return (await res.json()) as Promise; - } else { - throw new Error( - `GET ${url} failed with status ${res.status}. Response body: ${await res.text()}`, - ); - } - }); + return this.httpService.get(fullUrl); } // Fetches pages from middleware until the page contains at least one entry that meets the condition From e28dd3a22c5c7cc9777499ad143285ce5a9e5f99 Mon Sep 17 00:00:00 2001 From: Timo Erdelt Date: Wed, 24 Apr 2024 16:56:15 +0200 Subject: [PATCH 04/10] test: fix existing tests --- src/clients/coinmarketcap-client.model.ts | 2 +- src/clients/coinmarketcap-client.service.ts | 4 +- ...o-history-importer-v2.service.spec.ts.snap | 14 ++-- ...y-info-history-importer-v2.service.spec.ts | 74 +++++++++++++------ ...uidity-info-history-importer-v2.service.ts | 7 +- src/tasks/tasks.service.spec.ts | 4 + ...nfo-history-v2-db.service.e2e-spec.ts.snap | 2 +- ...ity-info-history-v2-db.service.e2e-spec.ts | 2 +- .../pair-liquidity-info-history-mock-data.ts | 16 +++- 9 files changed, 84 insertions(+), 41 deletions(-) diff --git a/src/clients/coinmarketcap-client.model.ts b/src/clients/coinmarketcap-client.model.ts index ea5ba29..0b32d66 100644 --- a/src/clients/coinmarketcap-client.model.ts +++ b/src/clients/coinmarketcap-client.model.ts @@ -1,4 +1,4 @@ -export type AEQuoteData = { +export type aeUsdQuoteData = { 1700: { id: number; name: string; diff --git a/src/clients/coinmarketcap-client.service.ts b/src/clients/coinmarketcap-client.service.ts index 32b3476..cdd002c 100644 --- a/src/clients/coinmarketcap-client.service.ts +++ b/src/clients/coinmarketcap-client.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common'; import { RateLimiter } from 'limiter'; import { - AEQuoteData, + aeUsdQuoteData, CoinMarketCapResponse, } from '@/clients/coinmarketcap-client.model'; import { HttpService } from '@/clients/http.service'; @@ -28,7 +28,7 @@ export class CoinmarketcapClientService { await this.rateLimiter.removeTokens(1); const timeEnd = this.roundMicroBlockTimeDownTo5MinInterval(microBlockTime); const url = `https://pro-api.coinmarketcap.com/v3/cryptocurrency/quotes/historical?id=${this.AE_CURRENCY_ID}&interval=${this.INTERVAL}&count=${this.COUNT}&time_end=${timeEnd}`; - return this.get>(url); + return this.get>(url); } private async get(url: string): Promise { diff --git a/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer-v2.service.spec.ts.snap b/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer-v2.service.spec.ts.snap index d43b674..4c8d685 100644 --- a/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer-v2.service.spec.ts.snap +++ b/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer-v2.service.spec.ts.snap @@ -4,10 +4,10 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should import liquidit [ [ { + "aeUsdPrice": "0.050559", "deltaReserve0": "0", "deltaReserve1": "0", "eventType": "CreatePair", - "aeUsdPrice": "0", "height": 10000, "logIndex": 0, "microBlockHash": "mh_hash0", @@ -21,10 +21,10 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should import liquidit ], [ { + "aeUsdPrice": "0.050559", "deltaReserve0": "100", "deltaReserve1": "100", "eventType": "PairMint", - "aeUsdPrice": "0", "height": 10001, "logIndex": 2, "microBlockHash": "mh_hash1", @@ -38,10 +38,10 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should import liquidit ], [ { + "aeUsdPrice": "0.050559", "deltaReserve0": "100", "deltaReserve1": "100", "eventType": "Sync", - "aeUsdPrice": "0", "height": 20002, "logIndex": 1, "microBlockHash": "mh_hash2", @@ -55,10 +55,10 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should import liquidit ], [ { + "aeUsdPrice": "0.050559", "deltaReserve0": "1", "deltaReserve1": "-1", "eventType": "SwapTokens", - "aeUsdPrice": "0", "height": 30003, "logIndex": 2, "microBlockHash": "mh_hash3", @@ -72,10 +72,10 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should import liquidit ], [ { + "aeUsdPrice": "0.050559", "deltaReserve0": "-101", "deltaReserve1": "-99", "eventType": "PairBurn", - "aeUsdPrice": "0", "height": 30003, "logIndex": 2, "microBlockHash": "mh_hash3", @@ -94,10 +94,10 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should skip a log if t [ [ { + "aeUsdPrice": "0.050559", "deltaReserve0": "1", "deltaReserve1": "-1", "eventType": "SwapTokens", - "aeUsdPrice": "0", "height": 30003, "logIndex": 2, "microBlockHash": "mh_hash3", @@ -116,10 +116,10 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should catch and inser [ [ { + "aeUsdPrice": "0.050559", "deltaReserve0": "1", "deltaReserve1": "-1", "eventType": "SwapTokens", - "aeUsdPrice": "0", "height": 30003, "logIndex": 2, "microBlockHash": "mh_hash3", diff --git a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.spec.ts b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.spec.ts index 41ddf1c..d4db885 100644 --- a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.spec.ts +++ b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.spec.ts @@ -8,7 +8,9 @@ import { PairLiquidityInfoHistoryV2ErrorDbService } from '@/database/pair-liquid import { bigIntToDecimal } from '@/lib/utils'; import { PairLiquidityInfoHistoryImporterV2Service } from '@/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service'; import resetAllMocks = jest.resetAllMocks; +import { CoinmarketcapClientService } from '@/clients/coinmarketcap-client.service'; import { + coinmarketCapResponseAeUsdQuoteData, contractLog1, contractLog2, contractLog3, @@ -22,12 +24,6 @@ import { pairWithTokens, } from '@/test/mock-data/pair-liquidity-info-history-mock-data'; -const mockMdwClient = { - getContract: jest.fn(), - getMicroBlock: jest.fn(), - getContractLogsUntilCondition: jest.fn(), -}; - const mockPairDb = { getAll: jest.fn() }; const mockPairLiquidityInfoHistoryV2Db = { @@ -40,6 +36,16 @@ const mockPairLiquidityInfoHistoryV2ErrorDb = { upsert: jest.fn(), }; +const mockMdwClient = { + getContract: jest.fn(), + getMicroBlock: jest.fn(), + getContractLogsUntilCondition: jest.fn(), +}; + +const mockCoinmarketcapClient = { + getHistoricalPriceDataThrottled: jest.fn(), +}; + describe('PairLiquidityInfoHistoryImporterV2Service', () => { let service: PairLiquidityInfoHistoryImporterV2Service; let logSpy: jest.SpyInstance; @@ -49,7 +55,6 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { const module: TestingModule = await Test.createTestingModule({ providers: [ PairLiquidityInfoHistoryImporterV2Service, - { provide: MdwHttpClientService, useValue: mockMdwClient }, { provide: PairDbService, useValue: mockPairDb }, { provide: PairLiquidityInfoHistoryV2DbService, @@ -59,7 +64,12 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { provide: PairLiquidityInfoHistoryV2ErrorDbService, useValue: mockPairLiquidityInfoHistoryV2ErrorDb, }, + { provide: MdwHttpClientService, useValue: mockMdwClient }, SdkClientService, + { + provide: CoinmarketcapClientService, + useValue: mockCoinmarketcapClient, + }, ], }).compile(); service = module.get( @@ -85,6 +95,9 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { }); mockMdwClient.getContract.mockResolvedValue(pairContract); mockMdwClient.getMicroBlock.mockResolvedValue(initialMicroBlock); + mockCoinmarketcapClient.getHistoricalPriceDataThrottled.mockResolvedValue( + coinmarketCapResponseAeUsdQuoteData, + ); mockPairLiquidityInfoHistoryV2Db.upsert.mockResolvedValue(null); mockMdwClient.getContractLogsUntilCondition.mockResolvedValue([ contractLog1, @@ -101,9 +114,7 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { await service.import(); // Assertions - expect( - mockPairLiquidityInfoHistoryV2ErrorDb.getErrorWithinHours, - ).toHaveBeenCalledTimes(5); // Once for pair and 4 times for each inserted event + expect(errorSpy.mock.calls).toEqual([]); expect(logSpy.mock.calls).toEqual([ ['Started syncing pair liquidity info history.'], @@ -117,6 +128,10 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { ['Finished liquidity info history sync for all pairs.'], ]); + expect( + mockPairLiquidityInfoHistoryV2ErrorDb.getErrorWithinHours, + ).toHaveBeenCalledTimes(5); // Once for pair and 4 times for each inserted event + expect(mockPairLiquidityInfoHistoryV2Db.upsert).toHaveBeenCalledTimes(5); expect( mockPairLiquidityInfoHistoryV2Db.upsert.mock.calls, @@ -144,6 +159,8 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { await service.import(); // Assertions + expect(errorSpy.mock.calls).toEqual([]); + expect(logSpy.mock.calls).toEqual([ ['Started syncing pair liquidity info history.'], ['Syncing liquidity info history for 1 pairs.'], @@ -172,6 +189,9 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { mockPairLiquidityInfoHistoryV2Db.getLastlySyncedLogByPairId.mockResolvedValue( {}, ); + mockCoinmarketcapClient.getHistoricalPriceDataThrottled.mockResolvedValue( + coinmarketCapResponseAeUsdQuoteData, + ); mockPairLiquidityInfoHistoryV2Db.upsert.mockResolvedValue(null); mockMdwClient.getContractLogsUntilCondition.mockResolvedValue([ contractLog1, @@ -184,9 +204,7 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { await service.import(); // Assertions - expect( - mockPairLiquidityInfoHistoryV2ErrorDb.getErrorWithinHours, - ).toHaveBeenCalledTimes(3); // Once for pair and 2 times for each event + expect(errorSpy.mock.calls).toEqual([]); expect(logSpy.mock.calls).toEqual([ ['Started syncing pair liquidity info history.'], @@ -200,6 +218,10 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { ['Finished liquidity info history sync for all pairs.'], ]); + expect( + mockPairLiquidityInfoHistoryV2ErrorDb.getErrorWithinHours, + ).toHaveBeenCalledTimes(3); // Once for pair and 2 times for each event + expect( mockPairLiquidityInfoHistoryV2Db.upsert.mock.calls, ).toMatchSnapshot(); @@ -231,9 +253,9 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { await service.import(); // Assertions - expect(mockPairLiquidityInfoHistoryV2ErrorDb.upsert).toHaveBeenCalledWith( - error, - ); + expect(errorSpy.mock.calls).toEqual([ + [`Skipped pair. ${JSON.stringify(error)}`], + ]); expect(logSpy.mock.calls).toEqual([ ['Started syncing pair liquidity info history.'], @@ -241,9 +263,9 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { ['Finished liquidity info history sync for all pairs.'], ]); - expect(errorSpy.mock.calls).toEqual([ - [`Skipped pair. ${JSON.stringify(error)}`], - ]); + expect(mockPairLiquidityInfoHistoryV2ErrorDb.upsert).toHaveBeenCalledWith( + error, + ); }); it('should catch and insert an error on log level', async () => { @@ -269,6 +291,9 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { contractLog4, contractLog5, ]); + mockCoinmarketcapClient.getHistoricalPriceDataThrottled.mockResolvedValue( + coinmarketCapResponseAeUsdQuoteData, + ); mockPairLiquidityInfoHistoryV2Db.upsert.mockResolvedValue(null); mockPairLiquidityInfoHistoryV2ErrorDb.upsert.mockResolvedValue(null); @@ -276,9 +301,10 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { await service.import(); // Assertions - expect(mockPairLiquidityInfoHistoryV2ErrorDb.upsert).toHaveBeenCalledWith( - error, - ); + + expect(errorSpy.mock.calls).toEqual([ + [`Skipped log. ${JSON.stringify(error)}`], + ]); expect(logSpy.mock.calls).toEqual([ ['Started syncing pair liquidity info history.'], @@ -289,6 +315,10 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { ['Finished liquidity info history sync for all pairs.'], ]); + expect(mockPairLiquidityInfoHistoryV2ErrorDb.upsert).toHaveBeenCalledWith( + error, + ); + expect(errorSpy.mock.calls).toEqual([ [`Skipped log. ${JSON.stringify(error)}`], ]); diff --git a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.ts b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.ts index b258af7..2961198 100644 --- a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.ts +++ b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.ts @@ -67,7 +67,7 @@ export class PairLiquidityInfoHistoryImporterV2Service { `Syncing liquidity info history for ${pairsWithTokens.length} pairs.`, ); - for (const pairWithTokens of pairsWithTokens.slice(3, 4)) { + for (const pairWithTokens of pairsWithTokens) { try { // If an error occurred for this pair recently, skip pair const error = @@ -286,6 +286,7 @@ export class PairLiquidityInfoHistoryImporterV2Service { const microBlock = await this.mdwClient.getMicroBlock( pairContract.block_hash, ); + const aeUsdPrice = await this.fetchPrice(parseInt(microBlock.time)); await this.pairLiquidityInfoHistoryDb .upsert({ pairId: pairWithTokens.id, @@ -294,7 +295,7 @@ export class PairLiquidityInfoHistoryImporterV2Service { reserve1: bigIntToDecimal(0n), deltaReserve0: bigIntToDecimal(0n), deltaReserve1: bigIntToDecimal(0n), - aeUsdPrice: bigIntToDecimal(0n), + aeUsdPrice: numberToDecimal(aeUsdPrice), height: parseInt(microBlock.height), microBlockHash: microBlock.hash, microBlockTime: BigInt(microBlock.time), @@ -354,7 +355,7 @@ export class PairLiquidityInfoHistoryImporterV2Service { } } - private fetchPrice(microBlockTime: number): Promise { + private async fetchPrice(microBlockTime: number): Promise { return this.coinmarketcapClient .getHistoricalPriceDataThrottled(microBlockTime) .then((res) => res.data['1700'].quotes[0].quote.USD.price); diff --git a/src/tasks/tasks.service.spec.ts b/src/tasks/tasks.service.spec.ts index 32d5c68..b3dbec8 100644 --- a/src/tasks/tasks.service.spec.ts +++ b/src/tasks/tasks.service.spec.ts @@ -1,5 +1,7 @@ import { Test, TestingModule } from '@nestjs/testing'; +import { CoinmarketcapClientService } from '@/clients/coinmarketcap-client.service'; +import { HttpService } from '@/clients/http.service'; import { MdwHttpClientService } from '@/clients/mdw-http-client.service'; import { SdkClientService } from '@/clients/sdk-client.service'; import { PairDbService } from '@/database/pair/pair-db.service'; @@ -27,6 +29,8 @@ describe('TasksService', () => { PairLiquidityInfoHistoryImporterV2Service, PairLiquidityInfoHistoryValidatorService, PairLiquidityInfoHistoryValidatorV2Service, + HttpService, + CoinmarketcapClientService, MdwHttpClientService, SdkClientService, PairDbService, diff --git a/test/e2e/__snapshots__/pair-liquidity-info-history-v2-db.service.e2e-spec.ts.snap b/test/e2e/__snapshots__/pair-liquidity-info-history-v2-db.service.e2e-spec.ts.snap index d718b52..1d29cc4 100644 --- a/test/e2e/__snapshots__/pair-liquidity-info-history-v2-db.service.e2e-spec.ts.snap +++ b/test/e2e/__snapshots__/pair-liquidity-info-history-v2-db.service.e2e-spec.ts.snap @@ -2,10 +2,10 @@ exports[`PairLiquidityInfoHistoryV2DbService upsert should correctly upsert an existing entry 1`] = ` { + "aeUsdPrice": "0.060559", "deltaReserve0": "-500", "deltaReserve1": "-500", "eventType": "PairBurn", - "aeUsdPrice": "0", "height": 200002, "id": 222, "logIndex": 1, diff --git a/test/e2e/pair-liquidity-info-history-v2-db.service.e2e-spec.ts b/test/e2e/pair-liquidity-info-history-v2-db.service.e2e-spec.ts index 0f86da8..ed83b2f 100644 --- a/test/e2e/pair-liquidity-info-history-v2-db.service.e2e-spec.ts +++ b/test/e2e/pair-liquidity-info-history-v2-db.service.e2e-spec.ts @@ -60,7 +60,7 @@ describe('PairLiquidityInfoHistoryV2DbService', () => { reserve1: new Decimal(500), deltaReserve0: new Decimal(-500), deltaReserve1: new Decimal(-500), - aeUsdPrice: new Decimal(0), + aeUsdPrice: new Decimal(0.060559), height: 200002, microBlockHash: historyEntry2.microBlockHash, microBlockTime: 2000000000002n, diff --git a/test/mock-data/pair-liquidity-info-history-mock-data.ts b/test/mock-data/pair-liquidity-info-history-mock-data.ts index b61257e..5c2491f 100644 --- a/test/mock-data/pair-liquidity-info-history-mock-data.ts +++ b/test/mock-data/pair-liquidity-info-history-mock-data.ts @@ -70,7 +70,7 @@ export const historyEntry1: PairLiquidityInfoHistoryV2 = { reserve1: new Decimal(1000), deltaReserve0: new Decimal(1000), deltaReserve1: new Decimal(1000), - aeUsdPrice: new Decimal(0), + aeUsdPrice: new Decimal(0.050559), height: 100001, microBlockHash: 'mh_entry1', microBlockTime: 1000000000001n, @@ -89,7 +89,7 @@ export const historyEntry2: PairLiquidityInfoHistoryV2 = { reserve1: new Decimal(950), deltaReserve0: new Decimal(50), deltaReserve1: new Decimal(-50), - aeUsdPrice: new Decimal(0), + aeUsdPrice: new Decimal(0.050559), height: 200002, microBlockHash: 'mh_entry2', microBlockTime: 2000000000002n, @@ -108,7 +108,7 @@ export const historyEntry3: PairLiquidityInfoHistoryV2 = { reserve1: new Decimal(1000), deltaReserve0: new Decimal(1000), deltaReserve1: new Decimal(1000), - aeUsdPrice: new Decimal(0), + aeUsdPrice: new Decimal(0.050559), height: 300003, microBlockHash: 'mh_entry3', microBlockTime: 3000000000003n, @@ -127,7 +127,7 @@ export const historyEntry4: PairLiquidityInfoHistoryV2 = { reserve1: new Decimal(950), deltaReserve0: new Decimal(50), deltaReserve1: new Decimal(-50), - aeUsdPrice: new Decimal(0), + aeUsdPrice: new Decimal(0.050559), height: 300003, microBlockHash: 'mh_entry3', microBlockTime: 3000000000003n, @@ -255,3 +255,11 @@ export const contractLog8 = { height: '30003', log_idx: '3', }; + +export const coinmarketCapResponseAeUsdQuoteData = { + data: { + 1700: { + quotes: [{ quote: { USD: { price: 0.050559 } } }], + }, + }, +}; From 4bdac9f453d1d8d168094514606fc0f8566808b4 Mon Sep 17 00:00:00 2001 From: Timo Erdelt Date: Wed, 24 Apr 2024 17:27:24 +0200 Subject: [PATCH 05/10] test: add unit test for CoinmarketcapClient --- .../coinmarketcap-client.service.spec.ts | 73 +++++++++++++++++++ src/clients/coinmarketcap-client.service.ts | 6 +- 2 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 src/clients/coinmarketcap-client.service.spec.ts diff --git a/src/clients/coinmarketcap-client.service.spec.ts b/src/clients/coinmarketcap-client.service.spec.ts new file mode 100644 index 0000000..5b35970 --- /dev/null +++ b/src/clients/coinmarketcap-client.service.spec.ts @@ -0,0 +1,73 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { CoinmarketcapClientService } from '@/clients/coinmarketcap-client.service'; +import { HttpService } from '@/clients/http.service'; +import resetAllMocks = jest.resetAllMocks; +import { RateLimiter } from 'limiter'; + +import { coinmarketCapResponseAeUsdQuoteData } from '@/test/mock-data/pair-liquidity-info-history-mock-data'; + +const mockHttpService = { + get: jest.fn(), +}; + +describe('CoinmarketcapClientService', () => { + let service: CoinmarketcapClientService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + CoinmarketcapClientService, + { provide: HttpService, useValue: mockHttpService }, + ], + }).compile(); + service = module.get( + CoinmarketcapClientService, + ); + resetAllMocks(); + }); + + describe('getHistoricalPriceDataThrottled', () => { + it('should correctly calculate and fetch the latest 5 min interval for a given timestamp', async () => { + // Mock functions + mockHttpService.get.mockResolvedValue( + coinmarketCapResponseAeUsdQuoteData, + ); + // Call function + await service.getHistoricalPriceDataThrottled(1704203935123); + + // Assertions + expect(mockHttpService.get).toHaveBeenCalledWith( + 'https://pro-api.coinmarketcap.com/v3/cryptocurrency/quotes/historical?id=1700&interval=24h&count=1&time_end=2024-01-02T13:55:00.000Z', + expect.anything(), + ); + + await service.getHistoricalPriceDataThrottled(1704203614123); + + expect(mockHttpService.get).toHaveBeenCalledWith( + 'https://pro-api.coinmarketcap.com/v3/cryptocurrency/quotes/historical?id=1700&interval=24h&count=1&time_end=2024-01-02T13:50:00.000Z', + expect.anything(), + ); + }); + + it('should be throtteled to defined rate limit', async () => { + service['rateLimiter'] = new RateLimiter({ + tokensPerInterval: 2, + interval: 'minute', + }); + + // Mock + mockHttpService.get.mockResolvedValue({}); + + // Call function + service.getHistoricalPriceDataThrottled(1704203935123); + service.getHistoricalPriceDataThrottled(1704203935123); + service.getHistoricalPriceDataThrottled(1704203935123); + + await new Promise((res) => setTimeout(res, 200)); + + // Assertion + expect(mockHttpService.get).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/src/clients/coinmarketcap-client.service.ts b/src/clients/coinmarketcap-client.service.ts index cdd002c..6cb79d3 100644 --- a/src/clients/coinmarketcap-client.service.ts +++ b/src/clients/coinmarketcap-client.service.ts @@ -11,12 +11,12 @@ import { HttpService } from '@/clients/http.service'; export class CoinmarketcapClientService { constructor(private httpService: HttpService) {} - private readonly AE_CURRENCY_ID = 1700; - private readonly COUNT = 1; - private readonly INTERVAL = '24h'; private readonly AUTH_HEADER = { 'X-CMC_PRO_API_KEY': process.env.COIN_MARKET_CAP_API_KEY || '', }; + private readonly AE_CURRENCY_ID = 1700; + private readonly COUNT = 1; + private readonly INTERVAL = '24h'; private readonly CALLS_LIMIT = 28; private readonly CALL_INTERVAL = 'minute'; private rateLimiter = new RateLimiter({ From 72605bb801ae3594bf163b4299eb3199893b09ef Mon Sep 17 00:00:00 2001 From: Timo Erdelt Date: Wed, 24 Apr 2024 17:31:41 +0200 Subject: [PATCH 06/10] refactor: improved type and variable naming consistency --- src/clients/coinmarketcap-client.model.ts | 8 ++++---- src/clients/coinmarketcap-client.service.spec.ts | 4 ++-- src/clients/coinmarketcap-client.service.ts | 6 +++--- ...air-liquidity-info-history-importer-v2.service.spec.ts | 8 ++++---- test/mock-data/pair-liquidity-info-history-mock-data.ts | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/clients/coinmarketcap-client.model.ts b/src/clients/coinmarketcap-client.model.ts index 0b32d66..c003162 100644 --- a/src/clients/coinmarketcap-client.model.ts +++ b/src/clients/coinmarketcap-client.model.ts @@ -1,4 +1,4 @@ -export type aeUsdQuoteData = { +export type AeUsdQuoteData = { 1700: { id: number; name: string; @@ -27,12 +27,12 @@ export type aeUsdQuoteData = { }; }; -export type CoinMarketCapResponse = { - status: CoinMarketCapStatus; +export type CoinmarketcapResponse = { + status: CoinmarketcapStatus; data: T; }; -export type CoinMarketCapStatus = { +export type CoinmarketcapStatus = { timestamp: Date; error_code: number; error_message: string; diff --git a/src/clients/coinmarketcap-client.service.spec.ts b/src/clients/coinmarketcap-client.service.spec.ts index 5b35970..6713260 100644 --- a/src/clients/coinmarketcap-client.service.spec.ts +++ b/src/clients/coinmarketcap-client.service.spec.ts @@ -5,7 +5,7 @@ import { HttpService } from '@/clients/http.service'; import resetAllMocks = jest.resetAllMocks; import { RateLimiter } from 'limiter'; -import { coinmarketCapResponseAeUsdQuoteData } from '@/test/mock-data/pair-liquidity-info-history-mock-data'; +import { coinmarketcapResponseAeUsdQuoteData } from '@/test/mock-data/pair-liquidity-info-history-mock-data'; const mockHttpService = { get: jest.fn(), @@ -31,7 +31,7 @@ describe('CoinmarketcapClientService', () => { it('should correctly calculate and fetch the latest 5 min interval for a given timestamp', async () => { // Mock functions mockHttpService.get.mockResolvedValue( - coinmarketCapResponseAeUsdQuoteData, + coinmarketcapResponseAeUsdQuoteData, ); // Call function await service.getHistoricalPriceDataThrottled(1704203935123); diff --git a/src/clients/coinmarketcap-client.service.ts b/src/clients/coinmarketcap-client.service.ts index 6cb79d3..ff1ef0e 100644 --- a/src/clients/coinmarketcap-client.service.ts +++ b/src/clients/coinmarketcap-client.service.ts @@ -2,8 +2,8 @@ import { Injectable } from '@nestjs/common'; import { RateLimiter } from 'limiter'; import { - aeUsdQuoteData, - CoinMarketCapResponse, + AeUsdQuoteData, + CoinmarketcapResponse, } from '@/clients/coinmarketcap-client.model'; import { HttpService } from '@/clients/http.service'; @@ -28,7 +28,7 @@ export class CoinmarketcapClientService { await this.rateLimiter.removeTokens(1); const timeEnd = this.roundMicroBlockTimeDownTo5MinInterval(microBlockTime); const url = `https://pro-api.coinmarketcap.com/v3/cryptocurrency/quotes/historical?id=${this.AE_CURRENCY_ID}&interval=${this.INTERVAL}&count=${this.COUNT}&time_end=${timeEnd}`; - return this.get>(url); + return this.get>(url); } private async get(url: string): Promise { diff --git a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.spec.ts b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.spec.ts index d4db885..32dc5ce 100644 --- a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.spec.ts +++ b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.spec.ts @@ -10,7 +10,7 @@ import { PairLiquidityInfoHistoryImporterV2Service } from '@/tasks/pair-liquidit import resetAllMocks = jest.resetAllMocks; import { CoinmarketcapClientService } from '@/clients/coinmarketcap-client.service'; import { - coinmarketCapResponseAeUsdQuoteData, + coinmarketcapResponseAeUsdQuoteData, contractLog1, contractLog2, contractLog3, @@ -96,7 +96,7 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { mockMdwClient.getContract.mockResolvedValue(pairContract); mockMdwClient.getMicroBlock.mockResolvedValue(initialMicroBlock); mockCoinmarketcapClient.getHistoricalPriceDataThrottled.mockResolvedValue( - coinmarketCapResponseAeUsdQuoteData, + coinmarketcapResponseAeUsdQuoteData, ); mockPairLiquidityInfoHistoryV2Db.upsert.mockResolvedValue(null); mockMdwClient.getContractLogsUntilCondition.mockResolvedValue([ @@ -190,7 +190,7 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { {}, ); mockCoinmarketcapClient.getHistoricalPriceDataThrottled.mockResolvedValue( - coinmarketCapResponseAeUsdQuoteData, + coinmarketcapResponseAeUsdQuoteData, ); mockPairLiquidityInfoHistoryV2Db.upsert.mockResolvedValue(null); mockMdwClient.getContractLogsUntilCondition.mockResolvedValue([ @@ -292,7 +292,7 @@ describe('PairLiquidityInfoHistoryImporterV2Service', () => { contractLog5, ]); mockCoinmarketcapClient.getHistoricalPriceDataThrottled.mockResolvedValue( - coinmarketCapResponseAeUsdQuoteData, + coinmarketcapResponseAeUsdQuoteData, ); mockPairLiquidityInfoHistoryV2Db.upsert.mockResolvedValue(null); mockPairLiquidityInfoHistoryV2ErrorDb.upsert.mockResolvedValue(null); diff --git a/test/mock-data/pair-liquidity-info-history-mock-data.ts b/test/mock-data/pair-liquidity-info-history-mock-data.ts index 5c2491f..5103469 100644 --- a/test/mock-data/pair-liquidity-info-history-mock-data.ts +++ b/test/mock-data/pair-liquidity-info-history-mock-data.ts @@ -256,7 +256,7 @@ export const contractLog8 = { log_idx: '3', }; -export const coinmarketCapResponseAeUsdQuoteData = { +export const coinmarketcapResponseAeUsdQuoteData = { data: { 1700: { quotes: [{ quote: { USD: { price: 0.050559 } } }], From 5a02575ce48d80bb372dcb678d1a33223d55292c Mon Sep 17 00:00:00 2001 From: Timo Erdelt Date: Mon, 6 May 2024 14:39:08 +0200 Subject: [PATCH 07/10] test: improve robustness of CoinmarketcapClient spec --- src/clients/coinmarketcap-client.service.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clients/coinmarketcap-client.service.spec.ts b/src/clients/coinmarketcap-client.service.spec.ts index 6713260..25dd2f9 100644 --- a/src/clients/coinmarketcap-client.service.spec.ts +++ b/src/clients/coinmarketcap-client.service.spec.ts @@ -60,8 +60,8 @@ describe('CoinmarketcapClientService', () => { mockHttpService.get.mockResolvedValue({}); // Call function - service.getHistoricalPriceDataThrottled(1704203935123); - service.getHistoricalPriceDataThrottled(1704203935123); + await service.getHistoricalPriceDataThrottled(1704203935123); + await service.getHistoricalPriceDataThrottled(1704203935123); service.getHistoricalPriceDataThrottled(1704203935123); await new Promise((res) => setTimeout(res, 200)); From 47ce0c1613c505b8f1010c637f1ecac5602e79cb Mon Sep 17 00:00:00 2001 From: Timo Erdelt Date: Thu, 25 Apr 2024 14:15:36 +0200 Subject: [PATCH 08/10] refactor: remove v1 history and rename v2 history to just history --- package-lock.json | 2 +- .../20240423120012_history_v2/migration.sql | 48 --- .../20240425094014_history_v2/migration.sql | 49 +++ prisma/schema.prisma | 52 +-- src/api/api.model.ts | 1 + ...-liquidity-info-history.controller.spec.ts | 176 ++++----- .../pair-liquidity-info-history.controller.ts | 33 +- src/database/database.module.ts | 6 - ...liquidity-info-history-error-db.service.ts | 12 +- ...uidity-info-history-v2-error-db.service.ts | 56 --- .../pair-liquidity-info-history-db.service.ts | 71 ++-- ...ir-liquidity-info-history-v2-db.service.ts | 72 ---- ...nfo-history-importer.service.spec.ts.snap} | 6 +- ...y-info-history-importer-v2.service.spec.ts | 331 ---------------- ...uidity-info-history-importer-v2.service.ts | 367 ----------------- ...dity-info-history-importer.service.spec.ts | 368 +++++++++++++----- ...liquidity-info-history-importer.service.ts | 356 +++++++++++------ ...-info-history-validator-v2.service.spec.ts | 252 ------------ ...idity-info-history-validator-v2.service.ts | 106 ----- ...ity-info-history-validator.service.spec.ts | 262 ++++++++++--- ...iquidity-info-history-validator.service.ts | 84 ++-- src/tasks/tasks.module.ts | 4 - src/tasks/tasks.service.spec.ts | 32 +- src/tasks/tasks.service.ts | 48 +-- ...-info-history-db.service.e2e-spec.ts.snap} | 2 +- ...uidity-info-history-db.service.e2e-spec.ts | 248 ++++++------ ...-info-history-error-db.service.e2e-spec.ts | 28 +- ...ity-info-history-v2-db.service.e2e-spec.ts | 130 ------- ...fo-history-v2-error-db.service.e2e-spec.ts | 116 ------ .../pair-liquidity-info-history-mock-data.ts | 12 +- 30 files changed, 1141 insertions(+), 2189 deletions(-) delete mode 100644 prisma/migrations/20240423120012_history_v2/migration.sql create mode 100644 prisma/migrations/20240425094014_history_v2/migration.sql delete mode 100644 src/database/pair-liquidity-info-history-error/pair-liquidity-info-history-v2-error-db.service.ts delete mode 100644 src/database/pair-liquidity-info-history/pair-liquidity-info-history-v2-db.service.ts rename src/tasks/pair-liquidity-info-history-importer/__snapshots__/{pair-liquidity-info-history-importer-v2.service.spec.ts.snap => pair-liquidity-info-history-importer.service.spec.ts.snap} (89%) delete mode 100644 src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.spec.ts delete mode 100644 src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.ts delete mode 100644 src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator-v2.service.spec.ts delete mode 100644 src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator-v2.service.ts rename test/e2e/__snapshots__/{pair-liquidity-info-history-v2-db.service.e2e-spec.ts.snap => pair-liquidity-info-history-db.service.e2e-spec.ts.snap} (79%) delete mode 100644 test/e2e/pair-liquidity-info-history-v2-db.service.e2e-spec.ts delete mode 100644 test/e2e/pair-liquidity-info-history-v2-error-db.service.e2e-spec.ts diff --git a/package-lock.json b/package-lock.json index 4c0cf81..df86082 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "dex-backend", - "version": "1.2.0", + "version": "1.2.1", "hasInstallScript": true, "license": "UNLICENSED", "dependencies": { diff --git a/prisma/migrations/20240423120012_history_v2/migration.sql b/prisma/migrations/20240423120012_history_v2/migration.sql deleted file mode 100644 index 23c7bfe..0000000 --- a/prisma/migrations/20240423120012_history_v2/migration.sql +++ /dev/null @@ -1,48 +0,0 @@ --- CreateTable -CREATE TABLE "PairLiquidityInfoHistoryV2" ( - "id" SERIAL NOT NULL, - "pairId" INTEGER NOT NULL, - "eventType" TEXT NOT NULL, - "reserve0" DECIMAL(100,0) NOT NULL, - "reserve1" DECIMAL(100,0) NOT NULL, - "deltaReserve0" DECIMAL(100,0) NOT NULL, - "deltaReserve1" DECIMAL(100,0) NOT NULL, - "aeUsdPrice" DECIMAL(100,6) NOT NULL, - "height" INTEGER NOT NULL, - "microBlockHash" TEXT NOT NULL, - "microBlockTime" BIGINT NOT NULL, - "transactionHash" TEXT NOT NULL, - "transactionIndex" BIGINT NOT NULL, - "logIndex" INTEGER NOT NULL, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - - CONSTRAINT "PairLiquidityInfoHistoryV2_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "PairLiquidityInfoHistoryV2Error" ( - "id" SERIAL NOT NULL, - "pairId" INTEGER NOT NULL, - "microBlockHash" TEXT NOT NULL, - "transactionHash" TEXT NOT NULL, - "logIndex" INTEGER NOT NULL, - "error" TEXT NOT NULL, - "timesOccurred" INTEGER NOT NULL DEFAULT 1, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - - CONSTRAINT "PairLiquidityInfoHistoryV2Error_pkey" PRIMARY KEY ("id") -); - --- CreateIndex -CREATE UNIQUE INDEX "PairLiquidityInfoHistoryV2_pairId_microBlockHash_transactio_key" ON "PairLiquidityInfoHistoryV2"("pairId", "microBlockHash", "transactionHash", "logIndex"); - --- CreateIndex -CREATE UNIQUE INDEX "PairLiquidityInfoHistoryV2Error_pairId_microBlockHash_trans_key" ON "PairLiquidityInfoHistoryV2Error"("pairId", "microBlockHash", "transactionHash", "logIndex", "error"); - --- AddForeignKey -ALTER TABLE "PairLiquidityInfoHistoryV2" ADD CONSTRAINT "PairLiquidityInfoHistoryV2_pairId_fkey" FOREIGN KEY ("pairId") REFERENCES "Pair"("id") ON DELETE RESTRICT ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "PairLiquidityInfoHistoryV2Error" ADD CONSTRAINT "PairLiquidityInfoHistoryV2Error_pairId_fkey" FOREIGN KEY ("pairId") REFERENCES "Pair"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/prisma/migrations/20240425094014_history_v2/migration.sql b/prisma/migrations/20240425094014_history_v2/migration.sql new file mode 100644 index 0000000..626bcba --- /dev/null +++ b/prisma/migrations/20240425094014_history_v2/migration.sql @@ -0,0 +1,49 @@ +/* + Warnings: + + - You are about to drop the column `totalSupply` on the `PairLiquidityInfoHistory` table. All the data in the column will be lost. + - A unique constraint covering the columns `[pairId,microBlockHash,transactionHash,logIndex]` on the table `PairLiquidityInfoHistory` will be added. If there are existing duplicate values, this will fail. + - A unique constraint covering the columns `[pairId,microBlockHash,transactionHash,logIndex,error]` on the table `PairLiquidityInfoHistoryError` will be added. If there are existing duplicate values, this will fail. + - Added the required column `aeUsdPrice` to the `PairLiquidityInfoHistory` table without a default value. This is not possible if the table is not empty. + - Added the required column `deltaReserve0` to the `PairLiquidityInfoHistory` table without a default value. This is not possible if the table is not empty. + - Added the required column `deltaReserve1` to the `PairLiquidityInfoHistory` table without a default value. This is not possible if the table is not empty. + - Added the required column `eventType` to the `PairLiquidityInfoHistory` table without a default value. This is not possible if the table is not empty. + - Added the required column `logIndex` to the `PairLiquidityInfoHistory` table without a default value. This is not possible if the table is not empty. + - Added the required column `transactionHash` to the `PairLiquidityInfoHistory` table without a default value. This is not possible if the table is not empty. + - Added the required column `transactionIndex` to the `PairLiquidityInfoHistory` table without a default value. This is not possible if the table is not empty. + - Changed the type of `reserve0` on the `PairLiquidityInfoHistory` table. No cast exists, the column would be dropped and recreated, which cannot be done if there is data, since the column is required. + - Changed the type of `reserve1` on the `PairLiquidityInfoHistory` table. No cast exists, the column would be dropped and recreated, which cannot be done if there is data, since the column is required. + - Added the required column `logIndex` to the `PairLiquidityInfoHistoryError` table without a default value. This is not possible if the table is not empty. + - Added the required column `transactionHash` to the `PairLiquidityInfoHistoryError` table without a default value. This is not possible if the table is not empty. + +*/ +-- DropIndex +DROP INDEX "PairLiquidityInfoHistory_pairId_microBlockHash_key"; + +-- DropIndex +DROP INDEX "PairLiquidityInfoHistoryError_pairId_microBlockHash_error_key"; + +-- AlterTable +ALTER TABLE "PairLiquidityInfoHistory" DROP COLUMN "totalSupply", +ADD COLUMN "aeUsdPrice" DECIMAL(100,6) NOT NULL, +ADD COLUMN "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, +ADD COLUMN "deltaReserve0" DECIMAL(100,0) NOT NULL, +ADD COLUMN "deltaReserve1" DECIMAL(100,0) NOT NULL, +ADD COLUMN "eventType" TEXT NOT NULL, +ADD COLUMN "logIndex" INTEGER NOT NULL, +ADD COLUMN "transactionHash" TEXT NOT NULL, +ADD COLUMN "transactionIndex" BIGINT NOT NULL, +DROP COLUMN "reserve0", +ADD COLUMN "reserve0" DECIMAL(100,0) NOT NULL, +DROP COLUMN "reserve1", +ADD COLUMN "reserve1" DECIMAL(100,0) NOT NULL; + +-- AlterTable +ALTER TABLE "PairLiquidityInfoHistoryError" ADD COLUMN "logIndex" INTEGER NOT NULL, +ADD COLUMN "transactionHash" TEXT NOT NULL; + +-- CreateIndex +CREATE UNIQUE INDEX "PairLiquidityInfoHistory_pairId_microBlockHash_transactionH_key" ON "PairLiquidityInfoHistory"("pairId", "microBlockHash", "transactionHash", "logIndex"); + +-- CreateIndex +CREATE UNIQUE INDEX "PairLiquidityInfoHistoryError_pairId_microBlockHash_transac_key" ON "PairLiquidityInfoHistoryError"("pairId", "microBlockHash", "transactionHash", "logIndex", "error"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index f336a5c..85ef87e 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -25,18 +25,16 @@ model Token { } model Pair { - id Int @id @default(autoincrement()) - address String @unique - token0 Token @relation("Pair_Token0", fields: [t0], references: [id]) - t0 Int - token1 Token @relation("Pair_Token1", fields: [t1], references: [id]) - t1 Int - liquidityInfo PairLiquidityInfo? @relation("Pair_Info") - synchronized Boolean - liquidityInfoHistory PairLiquidityInfoHistory[] - liquidityInfoHistoryV2 PairLiquidityInfoHistoryV2[] - liquidityInfoHistoryError PairLiquidityInfoHistoryError[] - liquidityInfoHistoryV2Error PairLiquidityInfoHistoryV2Error[] + id Int @id @default(autoincrement()) + address String @unique + token0 Token @relation("Pair_Token0", fields: [t0], references: [id]) + t0 Int + token1 Token @relation("Pair_Token1", fields: [t1], references: [id]) + t1 Int + liquidityInfo PairLiquidityInfo? @relation("Pair_Info") + synchronized Boolean + liquidityInfoHistory PairLiquidityInfoHistory[] + liquidityInfoHistoryError PairLiquidityInfoHistoryError[] } model PairLiquidityInfo { @@ -49,34 +47,6 @@ model PairLiquidityInfo { } model PairLiquidityInfoHistory { - id Int @id @default(autoincrement()) - pair Pair @relation(fields: [pairId], references: [id]) - pairId Int - totalSupply String - reserve0 String - reserve1 String - height Int - microBlockHash String - microBlockTime BigInt - updatedAt DateTime @default(now()) @updatedAt - - @@unique(name: "pairIdMicroBlockHashUniqueIndex", [pairId, microBlockHash]) -} - -model PairLiquidityInfoHistoryError { - id Int @id @default(autoincrement()) - pair Pair @relation(fields: [pairId], references: [id]) - pairId Int - microBlockHash String - error String - timesOccurred Int @default(1) - createdAt DateTime @default(now()) - updatedAt DateTime @default(now()) @updatedAt - - @@unique(name: "pairIdMicroBlockHashErrorUniqueIndex", [pairId, microBlockHash, error]) -} - -model PairLiquidityInfoHistoryV2 { id Int @id @default(autoincrement()) pair Pair @relation(fields: [pairId], references: [id]) pairId Int @@ -98,7 +68,7 @@ model PairLiquidityInfoHistoryV2 { @@unique(name: "pairIdMicroBlockHashTxHashLogIndexUniqueIndex", [pairId, microBlockHash, transactionHash, logIndex]) } -model PairLiquidityInfoHistoryV2Error { +model PairLiquidityInfoHistoryError { id Int @id @default(autoincrement()) pair Pair @relation(fields: [pairId], references: [id]) pairId Int diff --git a/src/api/api.model.ts b/src/api/api.model.ts index 7e2afec..3dc47b9 100644 --- a/src/api/api.model.ts +++ b/src/api/api.model.ts @@ -258,6 +258,7 @@ export enum OrderQueryEnum { desc = 'desc', } +// TODO adjust export class PairLiquidityInfoHistoryEntry { @ApiProperty(pairAddressPropertyOptions) pairAddress: string; diff --git a/src/api/pair-liquidity-info-history/pair-liquidity-info-history.controller.spec.ts b/src/api/pair-liquidity-info-history/pair-liquidity-info-history.controller.spec.ts index f2bbcc1..83b02a9 100644 --- a/src/api/pair-liquidity-info-history/pair-liquidity-info-history.controller.spec.ts +++ b/src/api/pair-liquidity-info-history/pair-liquidity-info-history.controller.spec.ts @@ -1,6 +1,5 @@ import { INestApplication } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; -import { Pair, PairLiquidityInfoHistory } from '@prisma/client'; import * as request from 'supertest'; import { OrderQueryEnum } from '@/api/api.model'; @@ -34,93 +33,94 @@ describe('PairLiquidityInfoHistoryController', () => { jest.clearAllMocks(); }); - it('should return history entries and use default values for empty params', async () => { - // Mocks - const historyEntry1: { pair: Pair } & PairLiquidityInfoHistory = { - id: 1, - pairId: 1, - totalSupply: '2000148656239820912122563', - reserve0: '950875688379385634428666', - reserve1: '4208476309359648851631167', - height: 912485, - microBlockHash: 'mh_Tx43Gh3acudUNSUWihPcV1Se4XcoFK3aUFAtFZk2Z4Zv7igZs', - microBlockTime: 1709027642807n, - updatedAt: new Date('2024-03-20 17:04:51.625'), - pair: { - id: 1, - address: 'ct_efYtiwDg4YZxDWE3iLPzvrjb92CJPvzGwriv4ZRuvuTDMNMb9', - t0: 15, - t1: 5, - synchronized: true, - }, - }; - - const historyEntry2: { pair: Pair } & PairLiquidityInfoHistory = { - id: 2, - pairId: 3, - totalSupply: '9954575303087659158151', - reserve0: '20210309618736130321327', - reserve1: '4903471477408475598460', - height: 707395, - microBlockHash: 'mh_2dUTfmwFc2ymeroB534giVwEvsa8d44Vf8SXtvy6GeHjdgQoHj', - microBlockTime: 1671708830503n, - updatedAt: new Date('2024-03-20 12:16:49.065'), - pair: { - id: 3, - address: 'ct_22iY9F7hng23gN8awi4aGnLy54YSR41wztbqgQCquuLYvTiGcm', - t0: 17, - t1: 22, - synchronized: true, - }, - }; - - mockPairLiquidityInfoHistoryService.getAllHistoryEntries.mockResolvedValue( - [historyEntry1, historyEntry2], - ); - - // Call route - const result = await request(app.getHttpServer()).get( - '/history/liquidity', - ); - - // Assertions - expect( - mockPairLiquidityInfoHistoryService.getAllHistoryEntries, - ).toHaveBeenCalledWith( - 100, - 0, - OrderQueryEnum.asc, - undefined, - undefined, - undefined, - undefined, - ); - expect(result.status).toBe(200); - expect(result.body).toEqual([ - { - pairAddress: historyEntry1.pair.address, - liquidityInfo: { - totalSupply: historyEntry1.totalSupply, - reserve0: historyEntry1.reserve0, - reserve1: historyEntry1.reserve1, - }, - height: historyEntry1.height, - microBlockHash: historyEntry1.microBlockHash, - microBlockTime: historyEntry1.microBlockTime.toString(), - }, - { - pairAddress: historyEntry2.pair.address, - liquidityInfo: { - totalSupply: historyEntry2.totalSupply, - reserve0: historyEntry2.reserve0, - reserve1: historyEntry2.reserve1, - }, - height: historyEntry2.height, - microBlockHash: historyEntry2.microBlockHash, - microBlockTime: historyEntry2.microBlockTime.toString(), - }, - ]); - }); + // TODO fix with updated return + // it('should return history entries and use default values for empty params', async () => { + // // Mocks + // const historyEntry1: { pair: Pair } & PairLiquidityInfoHistory = { + // id: 1, + // pairId: 1, + // totalSupply: '2000148656239820912122563', + // reserve0: '950875688379385634428666', + // reserve1: '4208476309359648851631167', + // height: 912485, + // microBlockHash: 'mh_Tx43Gh3acudUNSUWihPcV1Se4XcoFK3aUFAtFZk2Z4Zv7igZs', + // microBlockTime: 1709027642807n, + // updatedAt: new Date('2024-03-20 17:04:51.625'), + // pair: { + // id: 1, + // address: 'ct_efYtiwDg4YZxDWE3iLPzvrjb92CJPvzGwriv4ZRuvuTDMNMb9', + // t0: 15, + // t1: 5, + // synchronized: true, + // }, + // }; + // + // const historyEntry2: { pair: Pair } & PairLiquidityInfoHistory = { + // id: 2, + // pairId: 3, + // totalSupply: '9954575303087659158151', + // reserve0: '20210309618736130321327', + // reserve1: '4903471477408475598460', + // height: 707395, + // microBlockHash: 'mh_2dUTfmwFc2ymeroB534giVwEvsa8d44Vf8SXtvy6GeHjdgQoHj', + // microBlockTime: 1671708830503n, + // updatedAt: new Date('2024-03-20 12:16:49.065'), + // pair: { + // id: 3, + // address: 'ct_22iY9F7hng23gN8awi4aGnLy54YSR41wztbqgQCquuLYvTiGcm', + // t0: 17, + // t1: 22, + // synchronized: true, + // }, + // }; + // + // mockPairLiquidityInfoHistoryService.getAllHistoryEntries.mockResolvedValue( + // [historyEntry1, historyEntry2], + // ); + // + // // Call route + // const result = await request(app.getHttpServer()).get( + // '/history/liquidity', + // ); + // + // // Assertions + // expect( + // mockPairLiquidityInfoHistoryService.getAllHistoryEntries, + // ).toHaveBeenCalledWith( + // 100, + // 0, + // OrderQueryEnum.asc, + // undefined, + // undefined, + // undefined, + // undefined, + // ); + // expect(result.status).toBe(200); + // expect(result.body).toEqual([ + // { + // pairAddress: historyEntry1.pair.address, + // liquidityInfo: { + // totalSupply: historyEntry1.totalSupply, + // reserve0: historyEntry1.reserve0, + // reserve1: historyEntry1.reserve1, + // }, + // height: historyEntry1.height, + // microBlockHash: historyEntry1.microBlockHash, + // microBlockTime: historyEntry1.microBlockTime.toString(), + // }, + // { + // pairAddress: historyEntry2.pair.address, + // liquidityInfo: { + // totalSupply: historyEntry2.totalSupply, + // reserve0: historyEntry2.reserve0, + // reserve1: historyEntry2.reserve1, + // }, + // height: historyEntry2.height, + // microBlockHash: historyEntry2.microBlockHash, + // microBlockTime: historyEntry2.microBlockTime.toString(), + // }, + // ]); + // }); it('should parse all query params correctly', async () => { // Mocks diff --git a/src/api/pair-liquidity-info-history/pair-liquidity-info-history.controller.ts b/src/api/pair-liquidity-info-history/pair-liquidity-info-history.controller.ts index 795a786..2314d34 100644 --- a/src/api/pair-liquidity-info-history/pair-liquidity-info-history.controller.ts +++ b/src/api/pair-liquidity-info-history/pair-liquidity-info-history.controller.ts @@ -7,8 +7,7 @@ import { } from '@nestjs/common'; import { ApiOperation, ApiQuery, ApiResponse } from '@nestjs/swagger'; -import * as dto from '@/api/api.model'; -import { OrderQueryEnum } from '@/api/api.model'; +import { OrderQueryEnum, PairLiquidityInfoHistoryEntry } from '@/api/api.model'; import { PairLiquidityInfoHistoryService } from '@/api/pair-liquidity-info-history/pair-liquidity-info-history.service'; import { ContractAddress } from '@/clients/sdk-client.model'; @@ -68,7 +67,7 @@ export class PairLiquidityInfoHistoryController { 'Retrieve only history entries that are equal or older than the given micro block time', required: false, }) - @ApiResponse({ status: 200, type: [dto.PairLiquidityInfoHistoryEntry] }) + @ApiResponse({ status: 200, type: [PairLiquidityInfoHistoryEntry] }) async findAll( @Query('limit', new ParseIntPipe({ optional: true })) limit: number = 100, @Query('offset', new ParseIntPipe({ optional: true })) offset: number = 0, @@ -80,7 +79,7 @@ export class PairLiquidityInfoHistoryController { fromBlockTime?: number, @Query('to-block-time', new ParseIntPipe({ optional: true })) toBlockTime?: number, - ): Promise { + ): Promise { return this.pairLiquidityInfoHistoryService .getAllHistoryEntries( Number(limit), @@ -92,17 +91,23 @@ export class PairLiquidityInfoHistoryController { toBlockTime != null ? BigInt(toBlockTime) : undefined, ) .then((entries) => - entries.map((entry) => ({ - pairAddress: entry.pair.address, - liquidityInfo: { - totalSupply: entry.totalSupply, - reserve0: entry.reserve0, - reserve1: entry.reserve1, + entries.map(() => + // { + // pairAddress: entry.pair.address, + // liquidityInfo: { + // totalSupply: entry.totalSupply, + // reserve0: entry.reserve0, + // reserve1: entry.reserve1, + // }, + // height: entry.height, + // microBlockHash: entry.microBlockHash, + // microBlockTime: entry.microBlockTime.toString(), + // } + { + // TODO adjust + return {} as PairLiquidityInfoHistoryEntry; }, - height: entry.height, - microBlockHash: entry.microBlockHash, - microBlockTime: entry.microBlockTime.toString(), - })), + ), ); } } diff --git a/src/database/database.module.ts b/src/database/database.module.ts index 66a387d..0b27863 100644 --- a/src/database/database.module.ts +++ b/src/database/database.module.ts @@ -2,9 +2,7 @@ import { Module } from '@nestjs/common'; import { PairDbService } from '@/database/pair/pair-db.service'; import { PairLiquidityInfoHistoryDbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-db.service'; -import { PairLiquidityInfoHistoryV2DbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-v2-db.service'; import { PairLiquidityInfoHistoryErrorDbService } from '@/database/pair-liquidity-info-history-error/pair-liquidity-info-history-error-db.service'; -import { PairLiquidityInfoHistoryV2ErrorDbService } from '@/database/pair-liquidity-info-history-error/pair-liquidity-info-history-v2-error-db.service'; import { PrismaService } from '@/database/prisma.service'; import { TokenDbService } from '@/database/token/token-db.service'; @@ -14,16 +12,12 @@ import { TokenDbService } from '@/database/token/token-db.service'; PairDbService, PairLiquidityInfoHistoryDbService, PairLiquidityInfoHistoryErrorDbService, - PairLiquidityInfoHistoryV2DbService, - PairLiquidityInfoHistoryV2ErrorDbService, TokenDbService, ], exports: [ PairDbService, PairLiquidityInfoHistoryDbService, PairLiquidityInfoHistoryErrorDbService, - PairLiquidityInfoHistoryV2DbService, - PairLiquidityInfoHistoryV2ErrorDbService, TokenDbService, ], }) diff --git a/src/database/pair-liquidity-info-history-error/pair-liquidity-info-history-error-db.service.ts b/src/database/pair-liquidity-info-history-error/pair-liquidity-info-history-error-db.service.ts index edf5099..1cddf61 100644 --- a/src/database/pair-liquidity-info-history-error/pair-liquidity-info-history-error-db.service.ts +++ b/src/database/pair-liquidity-info-history-error/pair-liquidity-info-history-error-db.service.ts @@ -7,15 +7,19 @@ import { PrismaService } from '@/database/prisma.service'; export class PairLiquidityInfoHistoryErrorDbService { constructor(private prisma: PrismaService) {} - getErrorByPairIdAndMicroBlockHashWithinHours( + getErrorWithinHours( pairId: number, microBlockHash: string, + transactionHash: string, + logIndex: number, withinHours: number, ): Promise { return this.prisma.pairLiquidityInfoHistoryError.findFirst({ where: { pairId: pairId, microBlockHash: microBlockHash, + transactionHash: transactionHash, + logIndex: logIndex, updatedAt: { gt: new Date(Date.now() - withinHours * 60 * 60 * 1000), }, @@ -31,9 +35,11 @@ export class PairLiquidityInfoHistoryErrorDbService { ) { return this.prisma.pairLiquidityInfoHistoryError.upsert({ where: { - pairIdMicroBlockHashErrorUniqueIndex: { + pairIdMicroBlockHashTxHashLogIndexErrorUniqueIndex: { pairId: data.pairId, microBlockHash: data.microBlockHash, + transactionHash: data.transactionHash, + logIndex: data.logIndex, error: data.error, }, }, @@ -41,6 +47,8 @@ export class PairLiquidityInfoHistoryErrorDbService { create: { pairId: data.pairId, microBlockHash: data.microBlockHash, + transactionHash: data.transactionHash, + logIndex: data.logIndex, error: data.error, }, }); diff --git a/src/database/pair-liquidity-info-history-error/pair-liquidity-info-history-v2-error-db.service.ts b/src/database/pair-liquidity-info-history-error/pair-liquidity-info-history-v2-error-db.service.ts deleted file mode 100644 index eb76522..0000000 --- a/src/database/pair-liquidity-info-history-error/pair-liquidity-info-history-v2-error-db.service.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { PairLiquidityInfoHistoryV2Error } from '@prisma/client'; - -import { PrismaService } from '@/database/prisma.service'; - -@Injectable() -export class PairLiquidityInfoHistoryV2ErrorDbService { - constructor(private prisma: PrismaService) {} - - getErrorWithinHours( - pairId: number, - microBlockHash: string, - transactionHash: string, - logIndex: number, - withinHours: number, - ): Promise { - return this.prisma.pairLiquidityInfoHistoryV2Error.findFirst({ - where: { - pairId: pairId, - microBlockHash: microBlockHash, - transactionHash: transactionHash, - logIndex: logIndex, - updatedAt: { - gt: new Date(Date.now() - withinHours * 60 * 60 * 1000), - }, - }, - }); - } - - upsert( - data: Omit< - PairLiquidityInfoHistoryV2Error, - 'id' | 'timesOccurred' | 'createdAt' | 'updatedAt' - >, - ) { - return this.prisma.pairLiquidityInfoHistoryV2Error.upsert({ - where: { - pairIdMicroBlockHashTxHashLogIndexErrorUniqueIndex: { - pairId: data.pairId, - microBlockHash: data.microBlockHash, - transactionHash: data.transactionHash, - logIndex: data.logIndex, - error: data.error, - }, - }, - update: { timesOccurred: { increment: 1 } }, - create: { - pairId: data.pairId, - microBlockHash: data.microBlockHash, - transactionHash: data.transactionHash, - logIndex: data.logIndex, - error: data.error, - }, - }); - } -} diff --git a/src/database/pair-liquidity-info-history/pair-liquidity-info-history-db.service.ts b/src/database/pair-liquidity-info-history/pair-liquidity-info-history-db.service.ts index b3ef271..387253c 100644 --- a/src/database/pair-liquidity-info-history/pair-liquidity-info-history-db.service.ts +++ b/src/database/pair-liquidity-info-history/pair-liquidity-info-history-db.service.ts @@ -9,6 +9,23 @@ import { PrismaService } from '@/database/prisma.service'; export class PairLiquidityInfoHistoryDbService { constructor(private prisma: PrismaService) {} + upsert( + data: Omit, + ) { + return this.prisma.pairLiquidityInfoHistory.upsert({ + where: { + pairIdMicroBlockHashTxHashLogIndexUniqueIndex: { + pairId: data.pairId, + microBlockHash: data.microBlockHash, + transactionHash: data.transactionHash, + logIndex: data.logIndex, + }, + }, + update: data, + create: data, + }); + } + getAll = ( limit: number, offset: number, @@ -32,57 +49,49 @@ export class PairLiquidityInfoHistoryDbService { }, orderBy: order != null - ? { - microBlockTime: order, - } + ? [ + { microBlockTime: order }, + { transactionIndex: order }, + { logIndex: order }, + ] : {}, take: limit, skip: offset, }); - getCountByPairId(pairId: number) { - return this.prisma.pairLiquidityInfoHistory.count({ - where: { - pairId: pairId, - }, - }); - } - getLastlySyncedBlockByPairId(pairId: number) { + getLastlySyncedLogByPairId(pairId: number) { return this.prisma.pairLiquidityInfoHistory.findFirst({ where: { pairId, }, - orderBy: [{ height: 'desc' }, { microBlockTime: 'desc' }], - select: { - height: true, - microBlockTime: true, - }, + orderBy: [ + { microBlockTime: 'desc' }, + { transactionIndex: 'desc' }, + { logIndex: 'desc' }, + ], }); } - getWithinHeightSorted(heightLimit: number) { + getWithinHeightSortedWithPair(heightLimit: number) { return this.prisma.pairLiquidityInfoHistory.findMany({ where: { height: { gte: heightLimit, }, }, - orderBy: { - microBlockTime: 'asc', - }, - }); - } - - upsert(data: Omit) { - return this.prisma.pairLiquidityInfoHistory.upsert({ - where: { - pairIdMicroBlockHashUniqueIndex: { - pairId: data.pairId, - microBlockHash: data.microBlockHash, + orderBy: [ + { microBlockTime: 'asc' }, + { transactionIndex: 'asc' }, + { logIndex: 'asc' }, + ], + include: { + pair: { + include: { + token0: true, + token1: true, + }, }, }, - update: data, - create: data, }); } diff --git a/src/database/pair-liquidity-info-history/pair-liquidity-info-history-v2-db.service.ts b/src/database/pair-liquidity-info-history/pair-liquidity-info-history-v2-db.service.ts deleted file mode 100644 index 2677adc..0000000 --- a/src/database/pair-liquidity-info-history/pair-liquidity-info-history-v2-db.service.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { PairLiquidityInfoHistoryV2 } from '@prisma/client'; - -import { PrismaService } from '@/database/prisma.service'; - -@Injectable() -export class PairLiquidityInfoHistoryV2DbService { - constructor(private prisma: PrismaService) {} - - upsert( - data: Omit, - ) { - return this.prisma.pairLiquidityInfoHistoryV2.upsert({ - where: { - pairIdMicroBlockHashTxHashLogIndexUniqueIndex: { - pairId: data.pairId, - microBlockHash: data.microBlockHash, - transactionHash: data.transactionHash, - logIndex: data.logIndex, - }, - }, - update: data, - create: data, - }); - } - - getLastlySyncedLogByPairId(pairId: number) { - return this.prisma.pairLiquidityInfoHistoryV2.findFirst({ - where: { - pairId, - }, - orderBy: [ - { microBlockTime: 'desc' }, - { transactionIndex: 'desc' }, - { logIndex: 'desc' }, - ], - }); - } - - getWithinHeightSortedWithPair(heightLimit: number) { - return this.prisma.pairLiquidityInfoHistoryV2.findMany({ - where: { - height: { - gte: heightLimit, - }, - }, - orderBy: [ - { microBlockTime: 'asc' }, - { transactionIndex: 'asc' }, - { logIndex: 'asc' }, - ], - include: { - pair: { - include: { - token0: true, - token1: true, - }, - }, - }, - }); - } - - deleteFromMicroBlockTime(microBlockTime: bigint) { - return this.prisma.pairLiquidityInfoHistoryV2.deleteMany({ - where: { - microBlockTime: { - gte: microBlockTime, - }, - }, - }); - } -} diff --git a/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer-v2.service.spec.ts.snap b/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer.service.spec.ts.snap similarity index 89% rename from src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer-v2.service.spec.ts.snap rename to src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer.service.spec.ts.snap index 4c8d685..ac26fb2 100644 --- a/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer-v2.service.spec.ts.snap +++ b/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer.service.spec.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`PairLiquidityInfoHistoryImporterV2Service import should import liquidity correctly 1`] = ` +exports[`PairLiquidityInfoHistoryImporterService import should import liquidity correctly 1`] = ` [ [ { @@ -90,7 +90,7 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should import liquidit ] `; -exports[`PairLiquidityInfoHistoryImporterV2Service import should skip a log if there was a recent error 1`] = ` +exports[`PairLiquidityInfoHistoryImporterService import should skip a log if there was a recent error 1`] = ` [ [ { @@ -112,7 +112,7 @@ exports[`PairLiquidityInfoHistoryImporterV2Service import should skip a log if t ] `; -exports[`PairLiquidityInfoHistoryImporterV2Service import should catch and insert an error on log level 1`] = ` +exports[`PairLiquidityInfoHistoryImporterService import should catch and insert an error on log level 1`] = ` [ [ { diff --git a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.spec.ts b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.spec.ts deleted file mode 100644 index 32dc5ce..0000000 --- a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.spec.ts +++ /dev/null @@ -1,331 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { MdwHttpClientService } from '@/clients/mdw-http-client.service'; -import { SdkClientService } from '@/clients/sdk-client.service'; -import { PairDbService } from '@/database/pair/pair-db.service'; -import { PairLiquidityInfoHistoryV2DbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-v2-db.service'; -import { PairLiquidityInfoHistoryV2ErrorDbService } from '@/database/pair-liquidity-info-history-error/pair-liquidity-info-history-v2-error-db.service'; -import { bigIntToDecimal } from '@/lib/utils'; -import { PairLiquidityInfoHistoryImporterV2Service } from '@/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service'; -import resetAllMocks = jest.resetAllMocks; -import { CoinmarketcapClientService } from '@/clients/coinmarketcap-client.service'; -import { - coinmarketcapResponseAeUsdQuoteData, - contractLog1, - contractLog2, - contractLog3, - contractLog4, - contractLog5, - contractLog6, - contractLog7, - contractLog8, - initialMicroBlock, - pairContract, - pairWithTokens, -} from '@/test/mock-data/pair-liquidity-info-history-mock-data'; - -const mockPairDb = { getAll: jest.fn() }; - -const mockPairLiquidityInfoHistoryV2Db = { - getLastlySyncedLogByPairId: jest.fn(), - upsert: jest.fn(), -}; - -const mockPairLiquidityInfoHistoryV2ErrorDb = { - getErrorWithinHours: jest.fn(), - upsert: jest.fn(), -}; - -const mockMdwClient = { - getContract: jest.fn(), - getMicroBlock: jest.fn(), - getContractLogsUntilCondition: jest.fn(), -}; - -const mockCoinmarketcapClient = { - getHistoricalPriceDataThrottled: jest.fn(), -}; - -describe('PairLiquidityInfoHistoryImporterV2Service', () => { - let service: PairLiquidityInfoHistoryImporterV2Service; - let logSpy: jest.SpyInstance; - let errorSpy: jest.SpyInstance; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - PairLiquidityInfoHistoryImporterV2Service, - { provide: PairDbService, useValue: mockPairDb }, - { - provide: PairLiquidityInfoHistoryV2DbService, - useValue: mockPairLiquidityInfoHistoryV2Db, - }, - { - provide: PairLiquidityInfoHistoryV2ErrorDbService, - useValue: mockPairLiquidityInfoHistoryV2ErrorDb, - }, - { provide: MdwHttpClientService, useValue: mockMdwClient }, - SdkClientService, - { - provide: CoinmarketcapClientService, - useValue: mockCoinmarketcapClient, - }, - ], - }).compile(); - service = module.get( - PairLiquidityInfoHistoryImporterV2Service, - ); - logSpy = jest.spyOn(service.logger, 'log'); - errorSpy = jest.spyOn(service.logger, 'error'); - resetAllMocks(); - }); - - describe('import', () => { - it('should import liquidity correctly', async () => { - // Mock functions - mockPairDb.getAll.mockResolvedValue([pairWithTokens]); - mockPairLiquidityInfoHistoryV2ErrorDb.getErrorWithinHours.mockResolvedValue( - undefined, - ); - mockPairLiquidityInfoHistoryV2Db.getLastlySyncedLogByPairId - .mockResolvedValueOnce(undefined) - .mockResolvedValueOnce({ - reserve0: bigIntToDecimal(100n), - reserve1: bigIntToDecimal(100n), - }); - mockMdwClient.getContract.mockResolvedValue(pairContract); - mockMdwClient.getMicroBlock.mockResolvedValue(initialMicroBlock); - mockCoinmarketcapClient.getHistoricalPriceDataThrottled.mockResolvedValue( - coinmarketcapResponseAeUsdQuoteData, - ); - mockPairLiquidityInfoHistoryV2Db.upsert.mockResolvedValue(null); - mockMdwClient.getContractLogsUntilCondition.mockResolvedValue([ - contractLog1, - contractLog2, - contractLog3, - contractLog4, - contractLog5, - contractLog6, - contractLog7, - contractLog8, - ]); - - // Start import - await service.import(); - - // Assertions - expect(errorSpy.mock.calls).toEqual([]); - - expect(logSpy.mock.calls).toEqual([ - ['Started syncing pair liquidity info history.'], - ['Syncing liquidity info history for 1 pairs.'], - [ - `Inserted initial liquidity for pair ${pairWithTokens.id} ${pairWithTokens.address}.`, - ], - [ - `Completed sync for pair ${pairWithTokens.id} ${pairWithTokens.address}. Synced 4 log(s).`, - ], - ['Finished liquidity info history sync for all pairs.'], - ]); - - expect( - mockPairLiquidityInfoHistoryV2ErrorDb.getErrorWithinHours, - ).toHaveBeenCalledTimes(5); // Once for pair and 4 times for each inserted event - - expect(mockPairLiquidityInfoHistoryV2Db.upsert).toHaveBeenCalledTimes(5); - expect( - mockPairLiquidityInfoHistoryV2Db.upsert.mock.calls, - ).toMatchSnapshot(); - }); - - it('should skip a pair if there was a recent error', async () => { - // Mock functions - mockPairDb.getAll.mockResolvedValue([pairWithTokens]); - mockPairLiquidityInfoHistoryV2ErrorDb.getErrorWithinHours.mockResolvedValue( - { - id: 1, - pairId: 1, - microBlockHash: '', - transactionHash: '', - logIndex: -1, - error: 'error', - timesOccurred: 1, - createdAt: Date.now(), - updatedAt: Date.now(), - }, - ); - - // Start import - await service.import(); - - // Assertions - expect(errorSpy.mock.calls).toEqual([]); - - expect(logSpy.mock.calls).toEqual([ - ['Started syncing pair liquidity info history.'], - ['Syncing liquidity info history for 1 pairs.'], - [`Skipped pair ${pairWithTokens.id} due to recent error.`], - ['Finished liquidity info history sync for all pairs.'], - ]); - }); - - it('should skip a log if there was a recent error', async () => { - // Mock functions - mockPairDb.getAll.mockResolvedValue([pairWithTokens]); - mockPairLiquidityInfoHistoryV2ErrorDb.getErrorWithinHours - .mockResolvedValueOnce(undefined) - .mockResolvedValueOnce({ - id: 1, - pairId: 1, - microBlockHash: contractLog1.block_hash, - transactionHash: contractLog1.call_tx_hash, - logIndex: contractLog1.log_idx, - error: 'error', - timesOccurred: 1, - createdAt: Date.now(), - updatedAt: Date.now(), - }) - .mockResolvedValueOnce(undefined); - mockPairLiquidityInfoHistoryV2Db.getLastlySyncedLogByPairId.mockResolvedValue( - {}, - ); - mockCoinmarketcapClient.getHistoricalPriceDataThrottled.mockResolvedValue( - coinmarketcapResponseAeUsdQuoteData, - ); - mockPairLiquidityInfoHistoryV2Db.upsert.mockResolvedValue(null); - mockMdwClient.getContractLogsUntilCondition.mockResolvedValue([ - contractLog1, - contractLog2, - contractLog4, - contractLog5, - ]); - - // Start import - await service.import(); - - // Assertions - expect(errorSpy.mock.calls).toEqual([]); - - expect(logSpy.mock.calls).toEqual([ - ['Started syncing pair liquidity info history.'], - ['Syncing liquidity info history for 1 pairs.'], - [ - `Skipped log with block hash ${contractLog1.block_hash} tx hash ${contractLog1.call_tx_hash} and log index ${contractLog1.log_idx} due to recent error.`, - ], - [ - `Completed sync for pair ${pairWithTokens.id} ${pairWithTokens.address}. Synced 1 log(s).`, - ], - ['Finished liquidity info history sync for all pairs.'], - ]); - - expect( - mockPairLiquidityInfoHistoryV2ErrorDb.getErrorWithinHours, - ).toHaveBeenCalledTimes(3); // Once for pair and 2 times for each event - - expect( - mockPairLiquidityInfoHistoryV2Db.upsert.mock.calls, - ).toMatchSnapshot(); - }); - - it('should catch and insert an error on pair level', async () => { - const error = { - pairId: pairWithTokens.id, - microBlockHash: '', - transactionHash: '', - logIndex: -1, - error: 'Error: error', - }; - - // Mock functions - mockPairDb.getAll.mockResolvedValue([pairWithTokens]); - mockPairLiquidityInfoHistoryV2ErrorDb.getErrorWithinHours.mockResolvedValue( - undefined, - ); - mockPairLiquidityInfoHistoryV2Db.getLastlySyncedLogByPairId.mockResolvedValue( - {}, - ); - mockMdwClient.getContractLogsUntilCondition.mockRejectedValue( - new Error('error'), - ); - mockPairLiquidityInfoHistoryV2ErrorDb.upsert.mockResolvedValue(null); - - // Start import - await service.import(); - - // Assertions - expect(errorSpy.mock.calls).toEqual([ - [`Skipped pair. ${JSON.stringify(error)}`], - ]); - - expect(logSpy.mock.calls).toEqual([ - ['Started syncing pair liquidity info history.'], - ['Syncing liquidity info history for 1 pairs.'], - ['Finished liquidity info history sync for all pairs.'], - ]); - - expect(mockPairLiquidityInfoHistoryV2ErrorDb.upsert).toHaveBeenCalledWith( - error, - ); - }); - - it('should catch and insert an error on log level', async () => { - const error = { - pairId: pairWithTokens.id, - microBlockHash: contractLog3.block_hash, - transactionHash: contractLog3.call_tx_hash, - logIndex: parseInt(contractLog3.log_idx), - error: 'Error: error', - }; - - // Mock functions - mockPairDb.getAll.mockResolvedValue([pairWithTokens]); - mockPairLiquidityInfoHistoryV2ErrorDb.getErrorWithinHours.mockResolvedValue( - undefined, - ); - mockPairLiquidityInfoHistoryV2Db.getLastlySyncedLogByPairId - .mockResolvedValueOnce({}) - .mockRejectedValueOnce(new Error('error')) - .mockRejectedValueOnce(undefined); - mockMdwClient.getContractLogsUntilCondition.mockResolvedValue([ - contractLog3, - contractLog4, - contractLog5, - ]); - mockCoinmarketcapClient.getHistoricalPriceDataThrottled.mockResolvedValue( - coinmarketcapResponseAeUsdQuoteData, - ); - mockPairLiquidityInfoHistoryV2Db.upsert.mockResolvedValue(null); - mockPairLiquidityInfoHistoryV2ErrorDb.upsert.mockResolvedValue(null); - - // Start import - await service.import(); - - // Assertions - - expect(errorSpy.mock.calls).toEqual([ - [`Skipped log. ${JSON.stringify(error)}`], - ]); - - expect(logSpy.mock.calls).toEqual([ - ['Started syncing pair liquidity info history.'], - ['Syncing liquidity info history for 1 pairs.'], - [ - `Completed sync for pair ${pairWithTokens.id} ${pairWithTokens.address}. Synced 1 log(s).`, - ], - ['Finished liquidity info history sync for all pairs.'], - ]); - - expect(mockPairLiquidityInfoHistoryV2ErrorDb.upsert).toHaveBeenCalledWith( - error, - ); - - expect(errorSpy.mock.calls).toEqual([ - [`Skipped log. ${JSON.stringify(error)}`], - ]); - - expect( - mockPairLiquidityInfoHistoryV2Db.upsert.mock.calls, - ).toMatchSnapshot(); - }); - }); -}); diff --git a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.ts b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.ts deleted file mode 100644 index 2961198..0000000 --- a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service.ts +++ /dev/null @@ -1,367 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { PairLiquidityInfoHistoryV2 } from '@prisma/client'; -import { orderBy } from 'lodash'; - -import { CoinmarketcapClientService } from '@/clients/coinmarketcap-client.service'; -import { ContractLog } from '@/clients/mdw-http-client.model'; -import { MdwHttpClientService } from '@/clients/mdw-http-client.service'; -import { ContractAddress } from '@/clients/sdk-client.model'; -import { SdkClientService } from '@/clients/sdk-client.service'; -import { PairDbService, PairWithTokens } from '@/database/pair/pair-db.service'; -import { PairLiquidityInfoHistoryV2DbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-v2-db.service'; -import { PairLiquidityInfoHistoryV2ErrorDbService } from '@/database/pair-liquidity-info-history-error/pair-liquidity-info-history-v2-error-db.service'; -import { bigIntToDecimal, decimalToBigInt, numberToDecimal } from '@/lib/utils'; - -export enum EventType { - Sync = 'Sync', - SwapTokens = 'SwapTokens', - PairMint = 'PairMint', - PairBurn = 'PairBurn', -} - -type Event = SyncEvent | DeltaReserveEvent; - -type SyncEvent = { - eventType: EventType.Sync; - reserve0: bigint; - reserve1: bigint; -}; - -type DeltaReserveEvent = { - eventType: EventType.SwapTokens | EventType.PairMint | EventType.PairBurn; - deltaReserve0: bigint; - deltaReserve1: bigint; -}; - -@Injectable() -export class PairLiquidityInfoHistoryImporterV2Service { - constructor( - private pairDb: PairDbService, - private pairLiquidityInfoHistoryDb: PairLiquidityInfoHistoryV2DbService, - private pairLiquidityInfoHistoryErrorDb: PairLiquidityInfoHistoryV2ErrorDbService, - private mdwClient: MdwHttpClientService, - private sdkClient: SdkClientService, - private coinmarketcapClient: CoinmarketcapClientService, - ) {} - - readonly logger = new Logger(PairLiquidityInfoHistoryImporterV2Service.name); - - readonly WITHIN_HOURS_TO_SKIP_IF_ERROR = 6; - readonly SLIDING_WINDOW_BLOCKS = 10; - - private readonly SYNC_EVENT_HASH = - '6O232NLB36RGK54HEJPVDFJVCSIVFV29KPORC07CSSDARM7LV4L0===='; - private readonly SWAP_TOKENS_EVENT_HASH = - 'K39AB2I57LEUOUQ04LTEOMSJPJC3G9VGFRKVNJ5QLRMVCMDOPIMG===='; - private readonly PAIR_BURN_EVENT_HASH = - 'OIS2ALGSJ03MTP2BR5RBFL1GOUGESRVPGE58LGM0MVG9K3VAFKUG===='; - private readonly PAIR_MINT_EVENT_HASH = - 'L2BEDU7I5T8OSEUPB61900P8FJR637OE4MC4A9875C390RMQHSN0===='; - - async import() { - this.logger.log('Started syncing pair liquidity info history.'); - - // Fetch all pairs from DB - const pairsWithTokens = await this.pairDb.getAll(); - this.logger.log( - `Syncing liquidity info history for ${pairsWithTokens.length} pairs.`, - ); - - for (const pairWithTokens of pairsWithTokens) { - try { - // If an error occurred for this pair recently, skip pair - const error = - await this.pairLiquidityInfoHistoryErrorDb.getErrorWithinHours( - pairWithTokens.id, - '', - '', - -1, - this.WITHIN_HOURS_TO_SKIP_IF_ERROR, - ); - if (error) { - this.logger.log( - `Skipped pair ${pairWithTokens.id} due to recent error.`, - ); - continue; - } - - // Get current height - const currentHeight = await this.sdkClient.getHeight(); - - // Get lastly synced log - const lastSyncedLog = - await this.pairLiquidityInfoHistoryDb.getLastlySyncedLogByPairId( - pairWithTokens.id, - ); - - // If first sync (= no entries present yet for pair), insert initial liquidity - if (!lastSyncedLog) { - await this.insertInitialLiquidity(pairWithTokens); - } - const lastSyncedHeight = lastSyncedLog?.height ?? 0; - const lastSyncedBlockTime = lastSyncedLog?.microBlockTime ?? 0n; - const lastSyncedTxIndex = lastSyncedLog?.transactionIndex ?? 0n; - const lastSyncedLogIndex = lastSyncedLog?.logIndex ?? -1; - - // Determine which logs to sync based on the lastly synced log - // Strategy: - // 1. Always (re-)fetch everything within the 10 most recent key blocks (currentHeight - 10). - // 2. If the history is outdated, fetch everything since the lastly synced log - const isHistoryOutdated = - lastSyncedHeight < currentHeight - this.SLIDING_WINDOW_BLOCKS; - - // To make sure we get all desired logs, fetch all contract log pages - // until the page contains a non-desired entry - const fetchContractLogsLimit = (contractLog: ContractLog) => - isHistoryOutdated - ? BigInt(contractLog.block_time) < lastSyncedBlockTime - : parseInt(contractLog.height) < - currentHeight - this.SLIDING_WINDOW_BLOCKS; - - const pairContractLogs = - await this.mdwClient.getContractLogsUntilCondition( - fetchContractLogsLimit, - pairWithTokens.address as ContractAddress, - ); - - // Filter out all logs we don't want to insert (based on the strategy above) and sort the logs - // in ascending order - const relevantContractLogs = orderBy( - pairContractLogs.filter((contractLog: ContractLog) => - isHistoryOutdated - ? (BigInt(contractLog.call_txi) === lastSyncedTxIndex && - parseInt(contractLog.log_idx) > lastSyncedLogIndex) || - BigInt(contractLog.call_txi) > lastSyncedTxIndex - : parseInt(contractLog.height) >= - currentHeight - this.SLIDING_WINDOW_BLOCKS, - ), - ['block_time', 'call_txi', 'log_idx'], - ['asc', 'asc', 'asc'], - ); - - // Parse events from logs - const logsAndEvents = relevantContractLogs - .map((log) => ({ - log: log, - event: this.parseEvent(log), - })) - .filter( - (logAndEvent): logAndEvent is { log: ContractLog; event: Event } => - !!logAndEvent.event, - ); - - // Insert liquidity info for events - let numUpserted = 0; - for (const current of logsAndEvents) { - try { - const succeeding = - logsAndEvents[logsAndEvents.indexOf(current) + 1]; - - let liquidityInfo: Omit< - PairLiquidityInfoHistoryV2, - 'id' | 'updatedAt' | 'createdAt' - >; - // If current event is a Sync event and the next event is not a Sync event, insert merged liquidity info - if ( - current.event.eventType === EventType.Sync && - succeeding.event.eventType !== EventType.Sync - ) { - const aeUsdPrice = await this.fetchPrice( - parseInt(succeeding.log.block_time), - ); - liquidityInfo = { - pairId: pairWithTokens.id, - eventType: succeeding.event.eventType, - reserve0: bigIntToDecimal(current.event.reserve0), - reserve1: bigIntToDecimal(current.event.reserve1), - deltaReserve0: bigIntToDecimal(succeeding.event.deltaReserve0), - deltaReserve1: bigIntToDecimal(succeeding.event.deltaReserve1), - aeUsdPrice: numberToDecimal(aeUsdPrice), - height: parseInt(succeeding.log.height), - microBlockHash: succeeding.log.block_hash, - microBlockTime: BigInt(succeeding.log.block_time), - transactionHash: succeeding.log.call_tx_hash, - transactionIndex: BigInt(succeeding.log.call_txi), - logIndex: parseInt(succeeding.log.log_idx), - }; - // Else if current event is a Sync event and the next event is also a Sync event, insert Sync event - } else if (current.event.eventType === EventType.Sync) { - const lastSyncedLog = - await this.pairLiquidityInfoHistoryDb.getLastlySyncedLogByPairId( - pairWithTokens.id, - ); - const aeUsdPrice = await this.fetchPrice( - parseInt(current.log.block_time), - ); - liquidityInfo = { - pairId: pairWithTokens.id, - eventType: current.event.eventType, - reserve0: bigIntToDecimal(current.event.reserve0), - reserve1: bigIntToDecimal(current.event.reserve1), - // Calculated by the delta from lastly synced log's reserve0 - deltaReserve0: lastSyncedLog - ? bigIntToDecimal( - current.event.reserve0 - - decimalToBigInt(lastSyncedLog.reserve0), - ) - : bigIntToDecimal(0n), - // Calculated by the delta from lastly synced log's reserve1 - deltaReserve1: lastSyncedLog - ? bigIntToDecimal( - current.event.reserve1 - - decimalToBigInt(lastSyncedLog.reserve1), - ) - : bigIntToDecimal(0n), - aeUsdPrice: numberToDecimal(aeUsdPrice), - height: parseInt(current.log.height), - microBlockHash: current.log.block_hash, - microBlockTime: BigInt(current.log.block_time), - transactionHash: current.log.call_tx_hash, - transactionIndex: BigInt(current.log.call_txi), - logIndex: parseInt(current.log.log_idx), - }; - // Else continue, as every non-Sync event is preceded by a Sync event and thus already inserted previously - } else { - continue; - } - - // If an error occurred for this log recently, skip block - const error = - await this.pairLiquidityInfoHistoryErrorDb.getErrorWithinHours( - pairWithTokens.id, - current.log.block_hash, - current.log.call_tx_hash, - parseInt(current.log.log_idx), - this.WITHIN_HOURS_TO_SKIP_IF_ERROR, - ); - if (error) { - this.logger.log( - `Skipped log with block hash ${current.log.block_hash} tx hash ${current.log.call_tx_hash} and log index ${current.log.log_idx} due to recent error.`, - ); - continue; - } - - // Upsert liquidity - await this.pairLiquidityInfoHistoryDb - .upsert(liquidityInfo) - .then(() => numUpserted++); - } catch (error) { - const errorData = { - pairId: pairWithTokens.id, - microBlockHash: current.log.block_hash, - transactionHash: current.log.call_tx_hash, - logIndex: parseInt(current.log.log_idx), - error: error.toString(), - }; - this.logger.error(`Skipped log. ${JSON.stringify(errorData)}`); - await this.pairLiquidityInfoHistoryErrorDb.upsert(errorData); - } - } - - if (numUpserted > 0) { - this.logger.log( - `Completed sync for pair ${pairWithTokens.id} ${pairWithTokens.address}. Synced ${numUpserted} log(s).`, - ); - } - } catch (error) { - const errorData = { - pairId: pairWithTokens.id, - microBlockHash: '', - transactionHash: '', - logIndex: -1, - error: error.toString(), - }; - this.logger.error(`Skipped pair. ${JSON.stringify(errorData)}`); - await this.pairLiquidityInfoHistoryErrorDb.upsert(errorData); - } - } - - this.logger.log('Finished liquidity info history sync for all pairs.'); - } - - private async insertInitialLiquidity(pairWithTokens: PairWithTokens) { - const pairContract = await this.mdwClient.getContract( - pairWithTokens.address as ContractAddress, - ); - const microBlock = await this.mdwClient.getMicroBlock( - pairContract.block_hash, - ); - const aeUsdPrice = await this.fetchPrice(parseInt(microBlock.time)); - await this.pairLiquidityInfoHistoryDb - .upsert({ - pairId: pairWithTokens.id, - eventType: 'CreatePair', - reserve0: bigIntToDecimal(0n), - reserve1: bigIntToDecimal(0n), - deltaReserve0: bigIntToDecimal(0n), - deltaReserve1: bigIntToDecimal(0n), - aeUsdPrice: numberToDecimal(aeUsdPrice), - height: parseInt(microBlock.height), - microBlockHash: microBlock.hash, - microBlockTime: BigInt(microBlock.time), - transactionHash: pairContract.source_tx_hash, - transactionIndex: 0n, - logIndex: 0, - }) - .then(() => - this.logger.log( - `Inserted initial liquidity for pair ${pairWithTokens.id} ${pairWithTokens.address}.`, - ), - ); - } - - private parseEvent(log: ContractLog): Event | undefined { - const parseEventData = (data: string): bigint[] => { - return data.split('|').map((d) => BigInt(d)); - }; - - switch (log.event_hash) { - case this.SYNC_EVENT_HASH: - // Sync - // args: [balance0, balance1], data: empty - return { - eventType: EventType.Sync, - reserve0: BigInt(log.args[0]), - reserve1: BigInt(log.args[1]), - }; - case this.SWAP_TOKENS_EVENT_HASH: - // SwapTokens - // args: [_, _], data: [amount0In, amount1In, amount0Out, amount1Out] - const swapTokensData = parseEventData(log.data); - return { - eventType: EventType.SwapTokens, - deltaReserve0: swapTokensData[0] - swapTokensData[2], - deltaReserve1: swapTokensData[1] - swapTokensData[3], - }; - case this.PAIR_MINT_EVENT_HASH: - // PairMint - // args: [_, amount0, amount1], data: empty - return { - eventType: EventType.PairMint, - deltaReserve0: BigInt(log.args[1]), - deltaReserve1: BigInt(log.args[2]), - }; - case this.PAIR_BURN_EVENT_HASH: - // PairBurn - // args: [_, _], data: [amount0, amount1] - const pairBurnData = parseEventData(log.data); - return { - eventType: EventType.PairBurn, - deltaReserve0: 0n - pairBurnData[0], - deltaReserve1: 0n - pairBurnData[1], - }; - default: - return undefined; - } - } - - private async fetchPrice(microBlockTime: number): Promise { - return this.coinmarketcapClient - .getHistoricalPriceDataThrottled(microBlockTime) - .then((res) => res.data['1700'].quotes[0].quote.USD.price); - // .catch((err) => { - // console.log(err); - // return 0; - // }); - } -} diff --git a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.spec.ts b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.spec.ts index 1d4a976..abcccdc 100644 --- a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.spec.ts +++ b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.spec.ts @@ -1,44 +1,60 @@ import { Test, TestingModule } from '@nestjs/testing'; -import { Contract } from '@/clients/mdw-http-client.model'; import { MdwHttpClientService } from '@/clients/mdw-http-client.service'; -import { ContractAddress } from '@/clients/sdk-client.model'; import { SdkClientService } from '@/clients/sdk-client.service'; import { PairDbService } from '@/database/pair/pair-db.service'; import { PairLiquidityInfoHistoryDbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-db.service'; -import { PairLiquidityInfoHistoryErrorDbService } from '@/database/pair-liquidity-info-history-error/pair-liquidity-info-history-error-db.service'; +import { bigIntToDecimal } from '@/lib/utils'; import { PairLiquidityInfoHistoryImporterService } from '@/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service'; +import resetAllMocks = jest.resetAllMocks; +import { CoinmarketcapClientService } from '@/clients/coinmarketcap-client.service'; +import { PairLiquidityInfoHistoryErrorDbService } from '@/database/pair-liquidity-info-history-error/pair-liquidity-info-history-error-db.service'; +import { + coinmarketcapResponseAeUsdQuoteData, + contractLog1, + contractLog2, + contractLog3, + contractLog4, + contractLog5, + contractLog6, + contractLog7, + contractLog8, + initialMicroBlock, + pairContract, + pairWithTokens, +} from '@/test/mock-data/pair-liquidity-info-history-mock-data'; -const mockMdwClient = { - getContract: jest.fn(), - getMicroBlock: jest.fn(), - getContractLogsUntilCondition: jest.fn(), - getContractBalancesAtMicroBlockHash: jest.fn(), - getAccountBalanceForContractAtMicroBlockHash: jest.fn(), -}; - -const mockPairDb = { - getAll: jest.fn(), -}; +const mockPairDb = { getAll: jest.fn() }; const mockPairLiquidityInfoHistoryDb = { - getLastlySyncedBlockByPairId: jest.fn(), + getLastlySyncedLogByPairId: jest.fn(), upsert: jest.fn(), }; const mockPairLiquidityInfoHistoryErrorDb = { - getErrorByPairIdAndMicroBlockHashWithinHours: jest.fn(), + getErrorWithinHours: jest.fn(), + upsert: jest.fn(), +}; + +const mockMdwClient = { + getContract: jest.fn(), + getMicroBlock: jest.fn(), + getContractLogsUntilCondition: jest.fn(), +}; + +const mockCoinmarketcapClient = { + getHistoricalPriceDataThrottled: jest.fn(), }; describe('PairLiquidityInfoHistoryImporterService', () => { let service: PairLiquidityInfoHistoryImporterService; + let logSpy: jest.SpyInstance; + let errorSpy: jest.SpyInstance; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ providers: [ PairLiquidityInfoHistoryImporterService, - SdkClientService, - { provide: MdwHttpClientService, useValue: mockMdwClient }, { provide: PairDbService, useValue: mockPairDb }, { provide: PairLiquidityInfoHistoryDbService, @@ -48,114 +64,268 @@ describe('PairLiquidityInfoHistoryImporterService', () => { provide: PairLiquidityInfoHistoryErrorDbService, useValue: mockPairLiquidityInfoHistoryErrorDb, }, + { provide: MdwHttpClientService, useValue: mockMdwClient }, + SdkClientService, + { + provide: CoinmarketcapClientService, + useValue: mockCoinmarketcapClient, + }, ], }).compile(); service = module.get( PairLiquidityInfoHistoryImporterService, ); + logSpy = jest.spyOn(service.logger, 'log'); + errorSpy = jest.spyOn(service.logger, 'error'); + resetAllMocks(); }); describe('import', () => { it('should import liquidity correctly', async () => { - // Mock data - const pair1 = { - id: 1, - address: 'ct_pair' as ContractAddress, - token0: { address: 'ct_token0' }, - token1: { address: 'ct_token1' }, - }; - const pair1Contract: Contract = { - aexn_type: '', - block_hash: 'mh_hash0', - contract: pair1.address, - source_tx_hash: 'th_', - source_tx_type: '', - create_tx: {}, - }; - const initialMicroBlock = { - hash: pair1Contract.block_hash, - height: '10000', - time: '1000000000000', - }; - const pairContractLog1 = { - block_time: '1000000000001', - block_hash: 'mh_hash1', - height: '10001', - }; - const pairContractLog2 = { - block_time: '1000000000002', - block_hash: 'mh_hash2', - height: '10002', - }; - // Mock functions - mockPairDb.getAll.mockResolvedValue([pair1]); - mockPairLiquidityInfoHistoryErrorDb.getErrorByPairIdAndMicroBlockHashWithinHours.mockResolvedValue( - undefined, - ); - mockPairLiquidityInfoHistoryDb.getLastlySyncedBlockByPairId.mockReturnValue( + mockPairDb.getAll.mockResolvedValue([pairWithTokens]); + mockPairLiquidityInfoHistoryErrorDb.getErrorWithinHours.mockResolvedValue( undefined, ); - mockMdwClient.getContract.mockResolvedValue(pair1Contract); + mockPairLiquidityInfoHistoryDb.getLastlySyncedLogByPairId + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce({ + reserve0: bigIntToDecimal(100n), + reserve1: bigIntToDecimal(100n), + }); + mockMdwClient.getContract.mockResolvedValue(pairContract); mockMdwClient.getMicroBlock.mockResolvedValue(initialMicroBlock); + mockCoinmarketcapClient.getHistoricalPriceDataThrottled.mockResolvedValue( + coinmarketcapResponseAeUsdQuoteData, + ); mockPairLiquidityInfoHistoryDb.upsert.mockResolvedValue(null); mockMdwClient.getContractLogsUntilCondition.mockResolvedValue([ - pairContractLog1, - pairContractLog2, + contractLog1, + contractLog2, + contractLog3, + contractLog4, + contractLog5, + contractLog6, + contractLog7, + contractLog8, + ]); + + // Start import + await service.import(); + + // Assertions + expect(errorSpy.mock.calls).toEqual([]); + + expect(logSpy.mock.calls).toEqual([ + ['Started syncing pair liquidity info history.'], + ['Syncing liquidity info history for 1 pairs.'], + [ + `Inserted initial liquidity for pair ${pairWithTokens.id} ${pairWithTokens.address}.`, + ], + [ + `Completed sync for pair ${pairWithTokens.id} ${pairWithTokens.address}. Synced 4 log(s).`, + ], + ['Finished liquidity info history sync for all pairs.'], ]); - mockMdwClient.getContractBalancesAtMicroBlockHash.mockResolvedValue([ - { amount: '1' }, - { amount: '1' }, + + expect( + mockPairLiquidityInfoHistoryErrorDb.getErrorWithinHours, + ).toHaveBeenCalledTimes(5); // Once for pair and 4 times for each inserted event + + expect(mockPairLiquidityInfoHistoryDb.upsert).toHaveBeenCalledTimes(5); + expect( + mockPairLiquidityInfoHistoryDb.upsert.mock.calls, + ).toMatchSnapshot(); + }); + + it('should skip a pair if there was a recent error', async () => { + // Mock functions + mockPairDb.getAll.mockResolvedValue([pairWithTokens]); + mockPairLiquidityInfoHistoryErrorDb.getErrorWithinHours.mockResolvedValue( + { + id: 1, + pairId: 1, + microBlockHash: '', + transactionHash: '', + logIndex: -1, + error: 'error', + timesOccurred: 1, + createdAt: Date.now(), + updatedAt: Date.now(), + }, + ); + + // Start import + await service.import(); + + // Assertions + expect(errorSpy.mock.calls).toEqual([]); + + expect(logSpy.mock.calls).toEqual([ + ['Started syncing pair liquidity info history.'], + ['Syncing liquidity info history for 1 pairs.'], + [`Skipped pair ${pairWithTokens.id} due to recent error.`], + ['Finished liquidity info history sync for all pairs.'], ]); - mockMdwClient.getAccountBalanceForContractAtMicroBlockHash.mockResolvedValue( - { amount: '1' }, + }); + + it('should skip a log if there was a recent error', async () => { + // Mock functions + mockPairDb.getAll.mockResolvedValue([pairWithTokens]); + mockPairLiquidityInfoHistoryErrorDb.getErrorWithinHours + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce({ + id: 1, + pairId: 1, + microBlockHash: contractLog1.block_hash, + transactionHash: contractLog1.call_tx_hash, + logIndex: contractLog1.log_idx, + error: 'error', + timesOccurred: 1, + createdAt: Date.now(), + updatedAt: Date.now(), + }) + .mockResolvedValueOnce(undefined); + mockPairLiquidityInfoHistoryDb.getLastlySyncedLogByPairId.mockResolvedValue( + {}, ); - jest.spyOn(service.logger, 'log'); + mockCoinmarketcapClient.getHistoricalPriceDataThrottled.mockResolvedValue( + coinmarketcapResponseAeUsdQuoteData, + ); + mockPairLiquidityInfoHistoryDb.upsert.mockResolvedValue(null); + mockMdwClient.getContractLogsUntilCondition.mockResolvedValue([ + contractLog1, + contractLog2, + contractLog4, + contractLog5, + ]); // Start import await service.import(); // Assertions - // Insert initial liquidity - expect(mockPairLiquidityInfoHistoryDb.upsert).toHaveBeenCalledWith({ - pairId: pair1.id, - totalSupply: '0', - reserve0: '0', - reserve1: '0', - height: parseInt(initialMicroBlock.height), - microBlockHash: initialMicroBlock.hash, - microBlockTime: BigInt(initialMicroBlock.time), - }); + expect(errorSpy.mock.calls).toEqual([]); + + expect(logSpy.mock.calls).toEqual([ + ['Started syncing pair liquidity info history.'], + ['Syncing liquidity info history for 1 pairs.'], + [ + `Skipped log with block hash ${contractLog1.block_hash} tx hash ${contractLog1.call_tx_hash} and log index ${contractLog1.log_idx} due to recent error.`, + ], + [ + `Completed sync for pair ${pairWithTokens.id} ${pairWithTokens.address}. Synced 1 log(s).`, + ], + ['Finished liquidity info history sync for all pairs.'], + ]); + expect( - mockPairLiquidityInfoHistoryErrorDb.getErrorByPairIdAndMicroBlockHashWithinHours, - ).toHaveBeenCalledTimes(3); - expect(service.logger.log).toHaveBeenCalledWith( - `Started syncing pair ${pair1.id} ${pair1.address}. Need to sync 2 micro block(s). This can take some time.`, + mockPairLiquidityInfoHistoryErrorDb.getErrorWithinHours, + ).toHaveBeenCalledTimes(3); // Once for pair and 2 times for each event + + expect( + mockPairLiquidityInfoHistoryDb.upsert.mock.calls, + ).toMatchSnapshot(); + }); + + it('should catch and insert an error on pair level', async () => { + const error = { + pairId: pairWithTokens.id, + microBlockHash: '', + transactionHash: '', + logIndex: -1, + error: 'Error: error', + }; + + // Mock functions + mockPairDb.getAll.mockResolvedValue([pairWithTokens]); + mockPairLiquidityInfoHistoryErrorDb.getErrorWithinHours.mockResolvedValue( + undefined, ); - expect(mockPairLiquidityInfoHistoryDb.upsert).toHaveBeenCalledWith({ - pairId: pair1.id, - totalSupply: '2', - reserve0: '1', - reserve1: '1', - height: parseInt(pairContractLog1.height), - microBlockHash: pairContractLog1.block_hash, - microBlockTime: BigInt(pairContractLog1.block_time), - }); - expect(mockPairLiquidityInfoHistoryDb.upsert).toHaveBeenCalledWith({ - pairId: pair1.id, - totalSupply: '2', - reserve0: '1', - reserve1: '1', - height: parseInt(pairContractLog2.height), - microBlockHash: pairContractLog2.block_hash, - microBlockTime: BigInt(pairContractLog2.block_time), - }); - expect(service.logger.log).toHaveBeenCalledWith( - `Completed sync for pair ${pair1.id} ${pair1.address}. Synced 2 micro block(s).`, + mockPairLiquidityInfoHistoryDb.getLastlySyncedLogByPairId.mockResolvedValue( + {}, ); - expect(service.logger.log).toHaveBeenCalledWith( - 'Finished liquidity info history sync for all pairs.', + mockMdwClient.getContractLogsUntilCondition.mockRejectedValue( + new Error('error'), ); + mockPairLiquidityInfoHistoryErrorDb.upsert.mockResolvedValue(null); + + // Start import + await service.import(); + + // Assertions + expect(errorSpy.mock.calls).toEqual([ + [`Skipped pair. ${JSON.stringify(error)}`], + ]); + + expect(logSpy.mock.calls).toEqual([ + ['Started syncing pair liquidity info history.'], + ['Syncing liquidity info history for 1 pairs.'], + ['Finished liquidity info history sync for all pairs.'], + ]); + + expect(mockPairLiquidityInfoHistoryErrorDb.upsert).toHaveBeenCalledWith( + error, + ); + }); + + it('should catch and insert an error on log level', async () => { + const error = { + pairId: pairWithTokens.id, + microBlockHash: contractLog3.block_hash, + transactionHash: contractLog3.call_tx_hash, + logIndex: parseInt(contractLog3.log_idx), + error: 'Error: error', + }; + + // Mock functions + mockPairDb.getAll.mockResolvedValue([pairWithTokens]); + mockPairLiquidityInfoHistoryErrorDb.getErrorWithinHours.mockResolvedValue( + undefined, + ); + mockPairLiquidityInfoHistoryDb.getLastlySyncedLogByPairId + .mockResolvedValueOnce({}) + .mockRejectedValueOnce(new Error('error')) + .mockRejectedValueOnce(undefined); + mockMdwClient.getContractLogsUntilCondition.mockResolvedValue([ + contractLog3, + contractLog4, + contractLog5, + ]); + mockCoinmarketcapClient.getHistoricalPriceDataThrottled.mockResolvedValue( + coinmarketcapResponseAeUsdQuoteData, + ); + mockPairLiquidityInfoHistoryDb.upsert.mockResolvedValue(null); + mockPairLiquidityInfoHistoryErrorDb.upsert.mockResolvedValue(null); + + // Start import + await service.import(); + + // Assertions + + expect(errorSpy.mock.calls).toEqual([ + [`Skipped log. ${JSON.stringify(error)}`], + ]); + + expect(logSpy.mock.calls).toEqual([ + ['Started syncing pair liquidity info history.'], + ['Syncing liquidity info history for 1 pairs.'], + [ + `Completed sync for pair ${pairWithTokens.id} ${pairWithTokens.address}. Synced 1 log(s).`, + ], + ['Finished liquidity info history sync for all pairs.'], + ]); + + expect(mockPairLiquidityInfoHistoryErrorDb.upsert).toHaveBeenCalledWith( + error, + ); + + expect(errorSpy.mock.calls).toEqual([ + [`Skipped log. ${JSON.stringify(error)}`], + ]); + + expect( + mockPairLiquidityInfoHistoryDb.upsert.mock.calls, + ).toMatchSnapshot(); }); }); }); diff --git a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.ts b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.ts index 0497332..6d2dc8a 100644 --- a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.ts +++ b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.ts @@ -1,39 +1,65 @@ import { Injectable, Logger } from '@nestjs/common'; -import { isEqual, orderBy, uniqWith } from 'lodash'; +import { PairLiquidityInfoHistory } from '@prisma/client'; +import { orderBy } from 'lodash'; +import { CoinmarketcapClientService } from '@/clients/coinmarketcap-client.service'; import { ContractLog } from '@/clients/mdw-http-client.model'; import { MdwHttpClientService } from '@/clients/mdw-http-client.service'; -import { - ContractAddress, - contractAddrToAccountAddr, - MicroBlockHash, -} from '@/clients/sdk-client.model'; +import { ContractAddress } from '@/clients/sdk-client.model'; import { SdkClientService } from '@/clients/sdk-client.service'; import { PairDbService, PairWithTokens } from '@/database/pair/pair-db.service'; import { PairLiquidityInfoHistoryDbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-db.service'; import { PairLiquidityInfoHistoryErrorDbService } from '@/database/pair-liquidity-info-history-error/pair-liquidity-info-history-error-db.service'; +import { bigIntToDecimal, decimalToBigInt, numberToDecimal } from '@/lib/utils'; -type MicroBlock = { - hash: MicroBlockHash; - timestamp: bigint; - height: number; +export enum EventType { + Sync = 'Sync', + SwapTokens = 'SwapTokens', + PairMint = 'PairMint', + PairBurn = 'PairBurn', +} + +type Event = SyncEvent | DeltaReserveEvent; + +type SyncEvent = { + eventType: EventType.Sync; + reserve0: bigint; + reserve1: bigint; +}; + +type DeltaReserveEvent = { + eventType: EventType.SwapTokens | EventType.PairMint | EventType.PairBurn; + deltaReserve0: bigint; + deltaReserve1: bigint; }; @Injectable() export class PairLiquidityInfoHistoryImporterService { constructor( - private mdwClient: MdwHttpClientService, private pairDb: PairDbService, private pairLiquidityInfoHistoryDb: PairLiquidityInfoHistoryDbService, private pairLiquidityInfoHistoryErrorDb: PairLiquidityInfoHistoryErrorDbService, + private mdwClient: MdwHttpClientService, private sdkClient: SdkClientService, + private coinmarketcapClient: CoinmarketcapClientService, ) {} readonly logger = new Logger(PairLiquidityInfoHistoryImporterService.name); readonly WITHIN_HOURS_TO_SKIP_IF_ERROR = 6; + readonly SLIDING_WINDOW_BLOCKS = 10; + + private readonly SYNC_EVENT_HASH = + '6O232NLB36RGK54HEJPVDFJVCSIVFV29KPORC07CSSDARM7LV4L0===='; + private readonly SWAP_TOKENS_EVENT_HASH = + 'K39AB2I57LEUOUQ04LTEOMSJPJC3G9VGFRKVNJ5QLRMVCMDOPIMG===='; + private readonly PAIR_BURN_EVENT_HASH = + 'OIS2ALGSJ03MTP2BR5RBFL1GOUGESRVPGE58LGM0MVG9K3VAFKUG===='; + private readonly PAIR_MINT_EVENT_HASH = + 'L2BEDU7I5T8OSEUPB61900P8FJR637OE4MC4A9875C390RMQHSN0===='; + async import() { - this.logger.log(`Started syncing pair liquidity info history.`); + this.logger.log('Started syncing pair liquidity info history.'); // Fetch all pairs from DB const pairsWithTokens = await this.pairDb.getAll(); @@ -45,9 +71,11 @@ export class PairLiquidityInfoHistoryImporterService { try { // If an error occurred for this pair recently, skip pair const error = - await this.pairLiquidityInfoHistoryErrorDb.getErrorByPairIdAndMicroBlockHashWithinHours( + await this.pairLiquidityInfoHistoryErrorDb.getErrorWithinHours( pairWithTokens.id, '', + '', + -1, this.WITHIN_HOURS_TO_SKIP_IF_ERROR, ); if (error) { @@ -60,129 +88,187 @@ export class PairLiquidityInfoHistoryImporterService { // Get current height const currentHeight = await this.sdkClient.getHeight(); - // Get lastly synced block - const { - height: lastSyncedHeight, - microBlockTime: lastSyncedBlockTime, - } = (await this.pairLiquidityInfoHistoryDb.getLastlySyncedBlockByPairId( - pairWithTokens.id, - )) || { height: 0, microBlockTime: 0n }; + // Get lastly synced log + const lastSyncedLog = + await this.pairLiquidityInfoHistoryDb.getLastlySyncedLogByPairId( + pairWithTokens.id, + ); // If first sync (= no entries present yet for pair), insert initial liquidity - if (lastSyncedHeight === 0 && lastSyncedBlockTime === 0n) { + if (!lastSyncedLog) { await this.insertInitialLiquidity(pairWithTokens); } + const lastSyncedHeight = lastSyncedLog?.height ?? 0; + const lastSyncedBlockTime = lastSyncedLog?.microBlockTime ?? 0n; + const lastSyncedTxIndex = lastSyncedLog?.transactionIndex ?? 0n; + const lastSyncedLogIndex = lastSyncedLog?.logIndex ?? -1; - // Determine which micro blocks to sync based on the lastly synced block + // Determine which logs to sync based on the lastly synced log // Strategy: // 1. Always (re-)fetch everything within the 10 most recent key blocks (currentHeight - 10). - // 2. If the history is outdated, fetch everything since the lastly synced micro block - const isHistoryOutdated = lastSyncedHeight < currentHeight - 10; + // 2. If the history is outdated, fetch everything since the lastly synced log + const isHistoryOutdated = + lastSyncedHeight < currentHeight - this.SLIDING_WINDOW_BLOCKS; - const microBlocksToFetchFilter = (contractLog: ContractLog) => + // To make sure we get all desired logs, fetch all contract log pages + // until the page contains a non-desired entry + const fetchContractLogsLimit = (contractLog: ContractLog) => isHistoryOutdated - ? BigInt(contractLog.block_time) > lastSyncedBlockTime - : parseInt(contractLog.height) >= currentHeight - 10; - - // To make sure we get all desired micro blocks, fetch all contract log pages - // until the page contains a non-desired micro block - const fetchContractLogsFilter = (contractLog: ContractLog) => - isHistoryOutdated - ? BigInt(contractLog.block_time) <= lastSyncedBlockTime - : parseInt(contractLog.height) < currentHeight - 10; + ? BigInt(contractLog.block_time) < lastSyncedBlockTime + : parseInt(contractLog.height) < + currentHeight - this.SLIDING_WINDOW_BLOCKS; const pairContractLogs = await this.mdwClient.getContractLogsUntilCondition( - fetchContractLogsFilter, + fetchContractLogsLimit, pairWithTokens.address as ContractAddress, ); - // From the logs, get unique micro blocks in ascending order - const microBlocksToFetch = orderBy( - uniqWith( - pairContractLogs - .filter(microBlocksToFetchFilter) - .map((contractLog) => { - return { - hash: contractLog.block_hash, - timestamp: BigInt(contractLog.block_time), - height: parseInt(contractLog.height), - }; - }), - isEqual, + // Filter out all logs we don't want to insert (based on the strategy above) and sort the logs + // in ascending order + const relevantContractLogs = orderBy( + pairContractLogs.filter((contractLog: ContractLog) => + isHistoryOutdated + ? (BigInt(contractLog.call_txi) === lastSyncedTxIndex && + parseInt(contractLog.log_idx) > lastSyncedLogIndex) || + BigInt(contractLog.call_txi) > lastSyncedTxIndex + : parseInt(contractLog.height) >= + currentHeight - this.SLIDING_WINDOW_BLOCKS, ), - 'timestamp', - 'asc', + ['block_time', 'call_txi', 'log_idx'], + ['asc', 'asc', 'asc'], ); - if (microBlocksToFetch.length > 0) { - this.logger.log( - `Started syncing pair ${pairWithTokens.id} ${pairWithTokens.address}. Need to sync ${microBlocksToFetch.length} micro block(s). This can take some time.`, + // Parse events from logs + const logsAndEvents = relevantContractLogs + .map((log) => ({ + log: log, + event: this.parseEvent(log), + })) + .filter( + (logAndEvent): logAndEvent is { log: ContractLog; event: Event } => + !!logAndEvent.event, ); - } else { - this.logger.log( - `Pair ${pairWithTokens.id} ${pairWithTokens.address} is already up to date.`, - ); - } + // Insert liquidity info for events let numUpserted = 0; - // Fetch and insert liquidity (totalSupply, reserve0, reserve1) for every micro block - for (const block of microBlocksToFetch) { + for (const current of logsAndEvents) { try { - // If an error occurred for this block recently, skip block + const succeeding = + logsAndEvents[logsAndEvents.indexOf(current) + 1]; + + let liquidityInfo: Omit< + PairLiquidityInfoHistory, + 'id' | 'updatedAt' | 'createdAt' + >; + // If current event is a Sync event and the next event is not a Sync event, insert merged liquidity info + if ( + current.event.eventType === EventType.Sync && + succeeding.event.eventType !== EventType.Sync + ) { + const aeUsdPrice = await this.fetchPrice( + parseInt(succeeding.log.block_time), + ); + liquidityInfo = { + pairId: pairWithTokens.id, + eventType: succeeding.event.eventType, + reserve0: bigIntToDecimal(current.event.reserve0), + reserve1: bigIntToDecimal(current.event.reserve1), + deltaReserve0: bigIntToDecimal(succeeding.event.deltaReserve0), + deltaReserve1: bigIntToDecimal(succeeding.event.deltaReserve1), + aeUsdPrice: numberToDecimal(aeUsdPrice), + height: parseInt(succeeding.log.height), + microBlockHash: succeeding.log.block_hash, + microBlockTime: BigInt(succeeding.log.block_time), + transactionHash: succeeding.log.call_tx_hash, + transactionIndex: BigInt(succeeding.log.call_txi), + logIndex: parseInt(succeeding.log.log_idx), + }; + // Else if current event is a Sync event and the next event is also a Sync event, insert Sync event + } else if (current.event.eventType === EventType.Sync) { + const lastSyncedLog = + await this.pairLiquidityInfoHistoryDb.getLastlySyncedLogByPairId( + pairWithTokens.id, + ); + const aeUsdPrice = await this.fetchPrice( + parseInt(current.log.block_time), + ); + liquidityInfo = { + pairId: pairWithTokens.id, + eventType: current.event.eventType, + reserve0: bigIntToDecimal(current.event.reserve0), + reserve1: bigIntToDecimal(current.event.reserve1), + // Calculated by the delta from lastly synced log's reserve0 + deltaReserve0: lastSyncedLog + ? bigIntToDecimal( + current.event.reserve0 - + decimalToBigInt(lastSyncedLog.reserve0), + ) + : bigIntToDecimal(0n), + // Calculated by the delta from lastly synced log's reserve1 + deltaReserve1: lastSyncedLog + ? bigIntToDecimal( + current.event.reserve1 - + decimalToBigInt(lastSyncedLog.reserve1), + ) + : bigIntToDecimal(0n), + aeUsdPrice: numberToDecimal(aeUsdPrice), + height: parseInt(current.log.height), + microBlockHash: current.log.block_hash, + microBlockTime: BigInt(current.log.block_time), + transactionHash: current.log.call_tx_hash, + transactionIndex: BigInt(current.log.call_txi), + logIndex: parseInt(current.log.log_idx), + }; + // Else continue, as every non-Sync event is preceded by a Sync event and thus already inserted previously + } else { + continue; + } + + // If an error occurred for this log recently, skip block const error = - await this.pairLiquidityInfoHistoryErrorDb.getErrorByPairIdAndMicroBlockHashWithinHours( + await this.pairLiquidityInfoHistoryErrorDb.getErrorWithinHours( pairWithTokens.id, - block.hash, + current.log.block_hash, + current.log.call_tx_hash, + parseInt(current.log.log_idx), this.WITHIN_HOURS_TO_SKIP_IF_ERROR, ); if (error) { this.logger.log( - `Skipped micro block ${block.hash} due to recent error.`, + `Skipped log with block hash ${current.log.block_hash} tx hash ${current.log.call_tx_hash} and log index ${current.log.log_idx} due to recent error.`, ); continue; } - // Fetch liquidity - const liquidity = await this.getLiquidityForPairAtBlock( - pairWithTokens, - block, - ); - // Upsert liquidity await this.pairLiquidityInfoHistoryDb - .upsert({ - pairId: pairWithTokens.id, - totalSupply: liquidity.totalSupply.toString(), - reserve0: liquidity.reserve0, - reserve1: liquidity.reserve1, - height: block.height, - microBlockHash: block.hash, - microBlockTime: block.timestamp, - }) + .upsert(liquidityInfo) .then(() => numUpserted++); } catch (error) { const errorData = { pairId: pairWithTokens.id, - microBlockHash: block.hash, + microBlockHash: current.log.block_hash, + transactionHash: current.log.call_tx_hash, + logIndex: parseInt(current.log.log_idx), error: error.toString(), }; - this.logger.error( - `Skipped microBlock. ${JSON.stringify(errorData)}`, - ); + this.logger.error(`Skipped log. ${JSON.stringify(errorData)}`); await this.pairLiquidityInfoHistoryErrorDb.upsert(errorData); } } if (numUpserted > 0) { this.logger.log( - `Completed sync for pair ${pairWithTokens.id} ${pairWithTokens.address}. Synced ${numUpserted} micro block(s).`, + `Completed sync for pair ${pairWithTokens.id} ${pairWithTokens.address}. Synced ${numUpserted} log(s).`, ); } } catch (error) { const errorData = { pairId: pairWithTokens.id, microBlockHash: '', + transactionHash: '', + logIndex: -1, error: error.toString(), }; this.logger.error(`Skipped pair. ${JSON.stringify(errorData)}`); @@ -200,54 +286,82 @@ export class PairLiquidityInfoHistoryImporterService { const microBlock = await this.mdwClient.getMicroBlock( pairContract.block_hash, ); + const aeUsdPrice = await this.fetchPrice(parseInt(microBlock.time)); await this.pairLiquidityInfoHistoryDb .upsert({ pairId: pairWithTokens.id, - totalSupply: '0', - reserve0: '0', - reserve1: '0', + eventType: 'CreatePair', + reserve0: bigIntToDecimal(0n), + reserve1: bigIntToDecimal(0n), + deltaReserve0: bigIntToDecimal(0n), + deltaReserve1: bigIntToDecimal(0n), + aeUsdPrice: numberToDecimal(aeUsdPrice), height: parseInt(microBlock.height), microBlockHash: microBlock.hash, microBlockTime: BigInt(microBlock.time), + transactionHash: pairContract.source_tx_hash, + transactionIndex: 0n, + logIndex: 0, }) .then(() => this.logger.log( - `Inserted initial liquidity for pair ${pairWithTokens.address}.`, + `Inserted initial liquidity for pair ${pairWithTokens.id} ${pairWithTokens.address}.`, ), ); } - private async getLiquidityForPairAtBlock( - pairWithTokens: PairWithTokens, - block: MicroBlock, - ) { - // Total supply is the sum of all amounts of the pair contract's balances - const pairBalances = - await this.mdwClient.getContractBalancesAtMicroBlockHash( - pairWithTokens.address as ContractAddress, - block.hash, - ); - const totalSupply = pairBalances - .map((contractBalance) => BigInt(contractBalance.amount)) - .reduce((a, b) => a + b, 0n); - - // reserve0 is the balance of the pair contract's account of token0 - const reserve0 = ( - await this.mdwClient.getAccountBalanceForContractAtMicroBlockHash( - pairWithTokens.token0.address as ContractAddress, - contractAddrToAccountAddr(pairWithTokens.address as ContractAddress), - block.hash, - ) - ).amount; - - // reserve1 is the balance of the pair contract's account of token1 - const reserve1 = ( - await this.mdwClient.getAccountBalanceForContractAtMicroBlockHash( - pairWithTokens.token1.address as ContractAddress, - contractAddrToAccountAddr(pairWithTokens.address as ContractAddress), - block.hash, - ) - ).amount; - - return { totalSupply, reserve0, reserve1 }; + + private parseEvent(log: ContractLog): Event | undefined { + const parseEventData = (data: string): bigint[] => { + return data.split('|').map((d) => BigInt(d)); + }; + + switch (log.event_hash) { + case this.SYNC_EVENT_HASH: + // Sync + // args: [balance0, balance1], data: empty + return { + eventType: EventType.Sync, + reserve0: BigInt(log.args[0]), + reserve1: BigInt(log.args[1]), + }; + case this.SWAP_TOKENS_EVENT_HASH: + // SwapTokens + // args: [_, _], data: [amount0In, amount1In, amount0Out, amount1Out] + const swapTokensData = parseEventData(log.data); + return { + eventType: EventType.SwapTokens, + deltaReserve0: swapTokensData[0] - swapTokensData[2], + deltaReserve1: swapTokensData[1] - swapTokensData[3], + }; + case this.PAIR_MINT_EVENT_HASH: + // PairMint + // args: [_, amount0, amount1], data: empty + return { + eventType: EventType.PairMint, + deltaReserve0: BigInt(log.args[1]), + deltaReserve1: BigInt(log.args[2]), + }; + case this.PAIR_BURN_EVENT_HASH: + // PairBurn + // args: [_, _], data: [amount0, amount1] + const pairBurnData = parseEventData(log.data); + return { + eventType: EventType.PairBurn, + deltaReserve0: 0n - pairBurnData[0], + deltaReserve1: 0n - pairBurnData[1], + }; + default: + return undefined; + } + } + + private async fetchPrice(microBlockTime: number): Promise { + return this.coinmarketcapClient + .getHistoricalPriceDataThrottled(microBlockTime) + .then((res) => res.data['1700'].quotes[0].quote.USD.price); + // .catch((err) => { + // console.log(err); + // return 0; + // }); } } diff --git a/src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator-v2.service.spec.ts b/src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator-v2.service.spec.ts deleted file mode 100644 index f6b7fc9..0000000 --- a/src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator-v2.service.spec.ts +++ /dev/null @@ -1,252 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { MdwHttpClientService } from '@/clients/mdw-http-client.service'; -import { - AccountAddress, - ContractAddress, - MicroBlockHash, -} from '@/clients/sdk-client.model'; -import { SdkClientService } from '@/clients/sdk-client.service'; -import { PairLiquidityInfoHistoryV2DbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-v2-db.service'; -import { PairLiquidityInfoHistoryValidatorV2Service } from '@/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator-v2.service'; -import { - historyEntry1, - historyEntry2, - historyEntry3, - historyEntry4, - pairWithTokens, -} from '@/test/mock-data/pair-liquidity-info-history-mock-data'; -import resetAllMocks = jest.resetAllMocks; - -const mockPairLiquidityInfoHistoryV2Db = { - getWithinHeightSortedWithPair: jest.fn(), - deleteFromMicroBlockTime: jest.fn(), -}; - -const mockMdwClient = { - getAccountBalanceForContractAtMicroBlockHash: jest.fn(), -}; - -const mockSdkClient = { - getHeight: jest.fn(), -}; - -describe('PairLiquidityInfoHistoryValidatorV2Service', () => { - let service: PairLiquidityInfoHistoryValidatorV2Service; - let logSpy: jest.SpyInstance; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - PairLiquidityInfoHistoryValidatorV2Service, - SdkClientService, - { - provide: PairLiquidityInfoHistoryV2DbService, - useValue: mockPairLiquidityInfoHistoryV2Db, - }, - { provide: MdwHttpClientService, useValue: mockMdwClient }, - { provide: SdkClientService, useValue: mockSdkClient }, - ], - }).compile(); - service = module.get( - PairLiquidityInfoHistoryValidatorV2Service, - ); - logSpy = jest.spyOn(service.logger, 'log'); - resetAllMocks(); - }); - - describe('validate', () => { - it('should not delete entries if the mdw data matches with the last element of a local micro block', async () => { - // Mock functions - mockSdkClient.getHeight.mockResolvedValue(900000); - mockPairLiquidityInfoHistoryV2Db.getWithinHeightSortedWithPair.mockResolvedValue( - [ - { ...historyEntry2, pair: pairWithTokens }, - { ...historyEntry3, pair: pairWithTokens }, - { ...historyEntry4, pair: pairWithTokens }, - ], - ); - - mockMdwClient.getAccountBalanceForContractAtMicroBlockHash.mockImplementation( - ( - contractAddress: ContractAddress, - accountAddress: AccountAddress, - microBlockHash: MicroBlockHash, - ) => { - if (microBlockHash === historyEntry2.microBlockHash) { - if (contractAddress === pairWithTokens.token0.address) { - return { - amount: historyEntry2.reserve0, - }; - } else { - return { - amount: historyEntry2.reserve1, - }; - } - } else if (microBlockHash === historyEntry4.microBlockHash) { - if (contractAddress === pairWithTokens.token0.address) { - return { - amount: historyEntry4.reserve0, - }; - } else { - return { - amount: historyEntry4.reserve1, - }; - } - } else { - return {}; - } - }, - ); - - // Start validation - await service.validate(); - - // Assertions - expect( - mockPairLiquidityInfoHistoryV2Db.getWithinHeightSortedWithPair, - ).toHaveBeenCalledWith(899980); // Current height - VALIDATION_WINDOW_BLOCKS - - expect( - mockMdwClient.getAccountBalanceForContractAtMicroBlockHash, - ).toHaveBeenCalledTimes(4); - - expect( - mockPairLiquidityInfoHistoryV2Db.deleteFromMicroBlockTime, - ).toHaveBeenCalledTimes(0); - expect(logSpy.mock.calls).toEqual([ - ['Started validating pair liquidity info history.'], - ['No problems in pair liquidity info history found.'], - ['Finished validating pair liquidity info history.'], - ]); - }); - - it("should correctly delete entries if the mdw data doesn't match with the local data", async () => { - // Mock functions - mockSdkClient.getHeight.mockResolvedValue(900000); - mockPairLiquidityInfoHistoryV2Db.getWithinHeightSortedWithPair.mockResolvedValue( - [ - { ...historyEntry1, pair: pairWithTokens }, - { ...historyEntry2, pair: pairWithTokens }, - { ...historyEntry3, pair: pairWithTokens }, - { ...historyEntry4, pair: pairWithTokens }, - ], - ); - - mockMdwClient.getAccountBalanceForContractAtMicroBlockHash.mockImplementation( - ( - contractAddress: ContractAddress, - accountAddress: AccountAddress, - microBlockHash: MicroBlockHash, - ) => { - if (microBlockHash === historyEntry1.microBlockHash) { - if (contractAddress === pairWithTokens.token0.address) { - return { - amount: historyEntry1.reserve0, - }; - } else { - return { - amount: historyEntry1.reserve1, - }; - } - } else if (microBlockHash === historyEntry2.microBlockHash) { - if (contractAddress === pairWithTokens.token0.address) { - return { - amount: '123', - }; - } else { - return { - amount: '123', - }; - } - } else { - return {}; - } - }, - ); - - mockPairLiquidityInfoHistoryV2Db.deleteFromMicroBlockTime.mockResolvedValue( - { count: 3 }, - ); - - // Start validation - await service.validate(); - - // Assertions - expect( - mockMdwClient.getAccountBalanceForContractAtMicroBlockHash, - ).toHaveBeenCalledTimes(4); - - expect( - mockPairLiquidityInfoHistoryV2Db.deleteFromMicroBlockTime, - ).toHaveBeenCalledWith(historyEntry2.microBlockTime); - expect(logSpy.mock.calls).toEqual([ - ['Started validating pair liquidity info history.'], - [ - 'Found an inconsistency in pair liquidity info history. Deleted 3 entries.', - ], - ['Finished validating pair liquidity info history.'], - ]); - }); - - it('should correctly delete entries if mdw returns an error', async () => { - // Mock functions - mockSdkClient.getHeight.mockResolvedValue(900000); - mockPairLiquidityInfoHistoryV2Db.getWithinHeightSortedWithPair.mockResolvedValue( - [ - { ...historyEntry1, pair: pairWithTokens }, - { ...historyEntry2, pair: pairWithTokens }, - { ...historyEntry3, pair: pairWithTokens }, - { ...historyEntry4, pair: pairWithTokens }, - ], - ); - - mockMdwClient.getAccountBalanceForContractAtMicroBlockHash.mockImplementation( - ( - contractAddress: ContractAddress, - accountAddress: AccountAddress, - microBlockHash: MicroBlockHash, - ) => { - if (microBlockHash === historyEntry1.microBlockHash) { - if (contractAddress === pairWithTokens.token0.address) { - return { - amount: historyEntry1.reserve0, - }; - } else { - return { - amount: historyEntry1.reserve1, - }; - } - } else if (microBlockHash === historyEntry2.microBlockHash) { - if (contractAddress === pairWithTokens.token0.address) { - throw new Error('mdw error'); - } - } - }, - ); - - mockPairLiquidityInfoHistoryV2Db.deleteFromMicroBlockTime.mockResolvedValue( - { count: 3 }, - ); - - // Start validation - await service.validate(); - - // Assertions - expect( - mockMdwClient.getAccountBalanceForContractAtMicroBlockHash, - ).toHaveBeenCalledTimes(3); - - expect( - mockPairLiquidityInfoHistoryV2Db.deleteFromMicroBlockTime, - ).toHaveBeenCalledWith(historyEntry2.microBlockTime); - expect(logSpy.mock.calls).toEqual([ - ['Started validating pair liquidity info history.'], - [ - 'Found an inconsistency in pair liquidity info history. Deleted 3 entries.', - ], - ['Finished validating pair liquidity info history.'], - ]); - }); - }); -}); diff --git a/src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator-v2.service.ts b/src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator-v2.service.ts deleted file mode 100644 index 3fef232..0000000 --- a/src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator-v2.service.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { groupBy, last, map } from 'lodash'; - -import { MdwHttpClientService } from '@/clients/mdw-http-client.service'; -import { - ContractAddress, - contractAddrToAccountAddr, - MicroBlockHash, -} from '@/clients/sdk-client.model'; -import { SdkClientService } from '@/clients/sdk-client.service'; -import { PairLiquidityInfoHistoryV2DbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-v2-db.service'; -import { decimalToBigInt } from '@/lib/utils'; - -@Injectable() -export class PairLiquidityInfoHistoryValidatorV2Service { - constructor( - private pairLiquidityInfoHistoryDb: PairLiquidityInfoHistoryV2DbService, - private mdwClient: MdwHttpClientService, - private sdkClient: SdkClientService, - ) {} - - readonly logger = new Logger(PairLiquidityInfoHistoryValidatorV2Service.name); - - readonly VALIDATION_WINDOW_BLOCKS = 20; - - async validate() { - this.logger.log(`Started validating pair liquidity info history.`); - - // Get current height - const currentHeight = await this.sdkClient.getHeight(); - - // Get all liquidity entries greater or equal the current height minus VALIDATION_WINDOW_BLOCKS - // and take the last entry of every microBlock to get the final reserve in that microBlock - const liquidityEntriesWithinHeightSorted = map( - groupBy( - await this.pairLiquidityInfoHistoryDb.getWithinHeightSortedWithPair( - currentHeight - this.VALIDATION_WINDOW_BLOCKS, - ), - 'microBlockHash', - ), - (group) => last(group)!, - ); - - // If the reserves of a local microBlock do not match with the data from the middleware or the block does not exist, - // delete all logs in this block and all newer entries - let numDeleted = 0; - for (const liquidityEntry of liquidityEntriesWithinHeightSorted) { - let isError = false; - let mdwReserve0: bigint | undefined; - let mdwReserve1: bigint | undefined; - - try { - // reserve0 is the balance of the pair contract's account of token0 - mdwReserve0 = BigInt( - ( - await this.mdwClient.getAccountBalanceForContractAtMicroBlockHash( - liquidityEntry.pair.token0.address as ContractAddress, - contractAddrToAccountAddr( - liquidityEntry.pair.address as ContractAddress, - ), - liquidityEntry.microBlockHash as MicroBlockHash, - ) - ).amount, - ); - - // reserve1 is the balance of the pair contract's account of token1 - mdwReserve1 = BigInt( - ( - await this.mdwClient.getAccountBalanceForContractAtMicroBlockHash( - liquidityEntry.pair.token1.address as ContractAddress, - contractAddrToAccountAddr( - liquidityEntry.pair.address as ContractAddress, - ), - liquidityEntry.microBlockHash as MicroBlockHash, - ) - ).amount, - ); - } catch (e) { - this.logger.error(e); - isError = true; - } - if ( - isError || - decimalToBigInt(liquidityEntry.reserve0) !== mdwReserve0 || - decimalToBigInt(liquidityEntry.reserve1) !== mdwReserve1 - ) { - numDeleted = ( - await this.pairLiquidityInfoHistoryDb.deleteFromMicroBlockTime( - liquidityEntry.microBlockTime, - ) - ).count; - break; - } - } - - if (numDeleted > 0) { - this.logger.log( - `Found an inconsistency in pair liquidity info history. Deleted ${numDeleted} entries.`, - ); - } else { - this.logger.log('No problems in pair liquidity info history found.'); - } - - this.logger.log('Finished validating pair liquidity info history.'); - } -} diff --git a/src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator.service.spec.ts b/src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator.service.spec.ts index 1597ae1..c39e92c 100644 --- a/src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator.service.spec.ts +++ b/src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator.service.spec.ts @@ -1,116 +1,252 @@ import { Test, TestingModule } from '@nestjs/testing'; import { MdwHttpClientService } from '@/clients/mdw-http-client.service'; +import { + AccountAddress, + ContractAddress, + MicroBlockHash, +} from '@/clients/sdk-client.model'; import { SdkClientService } from '@/clients/sdk-client.service'; import { PairLiquidityInfoHistoryDbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-db.service'; import { PairLiquidityInfoHistoryValidatorService } from '@/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator.service'; +import { + historyEntry1, + historyEntry2, + historyEntry3, + historyEntry4, + pairWithTokens, +} from '@/test/mock-data/pair-liquidity-info-history-mock-data'; +import resetAllMocks = jest.resetAllMocks; + +const mockPairLiquidityInfoHistoryDb = { + getWithinHeightSortedWithPair: jest.fn(), + deleteFromMicroBlockTime: jest.fn(), +}; const mockMdwClient = { - getKeyBlockMicroBlocks: jest.fn(), + getAccountBalanceForContractAtMicroBlockHash: jest.fn(), }; -const mockPairLiquidityInfoHistoryDb = { - getWithinHeightSorted: jest.fn(), - deleteFromMicroBlockTime: jest.fn(), +const mockSdkClient = { + getHeight: jest.fn(), }; describe('PairLiquidityInfoHistoryValidatorService', () => { let service: PairLiquidityInfoHistoryValidatorService; + let logSpy: jest.SpyInstance; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ providers: [ PairLiquidityInfoHistoryValidatorService, SdkClientService, - { provide: MdwHttpClientService, useValue: mockMdwClient }, { provide: PairLiquidityInfoHistoryDbService, useValue: mockPairLiquidityInfoHistoryDb, }, + { provide: MdwHttpClientService, useValue: mockMdwClient }, + { provide: SdkClientService, useValue: mockSdkClient }, ], }).compile(); service = module.get( PairLiquidityInfoHistoryValidatorService, ); + logSpy = jest.spyOn(service.logger, 'log'); + resetAllMocks(); }); describe('validate', () => { - it('should validate history correctly', async () => { - // Mock data - const historyEntry1 = { - height: 10001, - microBlockTime: 1000000000001n, - microBlockHash: 'mh_hash1', - }; - const historyEntry2 = { - height: 10001, - microBlockTime: 1000000000002n, - microBlockHash: 'mh_hash2', - }; - const historyEntry3 = { - height: 10003, - microBlockTime: 1000000000003n, - microBlockHash: 'mh_hash3', - }; - const historyEntry4 = { - height: 10004, - microBlockTime: 1000000000004n, - microBlockHash: 'mh_hash4', - }; - const historyEntry5 = { - height: 10005, - microBlockTime: 1000000000005n, - microBlockHash: 'mh_hash5', - }; + it('should not delete entries if the mdw data matches with the last element of a local micro block', async () => { // Mock functions - mockPairLiquidityInfoHistoryDb.getWithinHeightSorted.mockResolvedValue([ - historyEntry1, - historyEntry2, - historyEntry3, - historyEntry4, + mockSdkClient.getHeight.mockResolvedValue(900000); + mockPairLiquidityInfoHistoryDb.getWithinHeightSortedWithPair.mockResolvedValue( + [ + { ...historyEntry2, pair: pairWithTokens }, + { ...historyEntry3, pair: pairWithTokens }, + { ...historyEntry4, pair: pairWithTokens }, + ], + ); + + mockMdwClient.getAccountBalanceForContractAtMicroBlockHash.mockImplementation( + ( + contractAddress: ContractAddress, + accountAddress: AccountAddress, + microBlockHash: MicroBlockHash, + ) => { + if (microBlockHash === historyEntry2.microBlockHash) { + if (contractAddress === pairWithTokens.token0.address) { + return { + amount: historyEntry2.reserve0, + }; + } else { + return { + amount: historyEntry2.reserve1, + }; + } + } else if (microBlockHash === historyEntry4.microBlockHash) { + if (contractAddress === pairWithTokens.token0.address) { + return { + amount: historyEntry4.reserve0, + }; + } else { + return { + amount: historyEntry4.reserve1, + }; + } + } else { + return {}; + } + }, + ); + + // Start validation + await service.validate(); + + // Assertions + expect( + mockPairLiquidityInfoHistoryDb.getWithinHeightSortedWithPair, + ).toHaveBeenCalledWith(899980); // Current height - VALIDATION_WINDOW_BLOCKS + + expect( + mockMdwClient.getAccountBalanceForContractAtMicroBlockHash, + ).toHaveBeenCalledTimes(4); + + expect( + mockPairLiquidityInfoHistoryDb.deleteFromMicroBlockTime, + ).toHaveBeenCalledTimes(0); + expect(logSpy.mock.calls).toEqual([ + ['Started validating pair liquidity info history.'], + ['No problems in pair liquidity info history found.'], + ['Finished validating pair liquidity info history.'], ]); - mockMdwClient.getKeyBlockMicroBlocks.mockImplementation( - (height: number) => { - if (height === historyEntry1.height) { - return [ - { hash: historyEntry1.microBlockHash }, - { hash: historyEntry2.microBlockHash }, - { hash: 'mh_xyz' }, - ]; - } else if (height === historyEntry3.height) { - return [{ hash: historyEntry3.microBlockHash }, { hash: 'mh_xyz' }]; + }); + + it("should correctly delete entries if the mdw data doesn't match with the local data", async () => { + // Mock functions + mockSdkClient.getHeight.mockResolvedValue(900000); + mockPairLiquidityInfoHistoryDb.getWithinHeightSortedWithPair.mockResolvedValue( + [ + { ...historyEntry1, pair: pairWithTokens }, + { ...historyEntry2, pair: pairWithTokens }, + { ...historyEntry3, pair: pairWithTokens }, + { ...historyEntry4, pair: pairWithTokens }, + ], + ); + + mockMdwClient.getAccountBalanceForContractAtMicroBlockHash.mockImplementation( + ( + contractAddress: ContractAddress, + accountAddress: AccountAddress, + microBlockHash: MicroBlockHash, + ) => { + if (microBlockHash === historyEntry1.microBlockHash) { + if (contractAddress === pairWithTokens.token0.address) { + return { + amount: historyEntry1.reserve0, + }; + } else { + return { + amount: historyEntry1.reserve1, + }; + } + } else if (microBlockHash === historyEntry2.microBlockHash) { + if (contractAddress === pairWithTokens.token0.address) { + return { + amount: '123', + }; + } else { + return { + amount: '123', + }; + } } else { - return []; + return {}; } }, ); + mockPairLiquidityInfoHistoryDb.deleteFromMicroBlockTime.mockResolvedValue( - { count: 2 }, + { count: 3 }, ); - jest.spyOn(service.logger, 'log'); // Start validation await service.validate(); // Assertions - expect(mockMdwClient.getKeyBlockMicroBlocks).toHaveBeenCalledWith( - historyEntry1.height, - ); - expect(mockMdwClient.getKeyBlockMicroBlocks).toHaveBeenCalledWith( - historyEntry3.height, + expect( + mockMdwClient.getAccountBalanceForContractAtMicroBlockHash, + ).toHaveBeenCalledTimes(4); + + expect( + mockPairLiquidityInfoHistoryDb.deleteFromMicroBlockTime, + ).toHaveBeenCalledWith(historyEntry2.microBlockTime); + expect(logSpy.mock.calls).toEqual([ + ['Started validating pair liquidity info history.'], + [ + 'Found an inconsistency in pair liquidity info history. Deleted 3 entries.', + ], + ['Finished validating pair liquidity info history.'], + ]); + }); + + it('should correctly delete entries if mdw returns an error', async () => { + // Mock functions + mockSdkClient.getHeight.mockResolvedValue(900000); + mockPairLiquidityInfoHistoryDb.getWithinHeightSortedWithPair.mockResolvedValue( + [ + { ...historyEntry1, pair: pairWithTokens }, + { ...historyEntry2, pair: pairWithTokens }, + { ...historyEntry3, pair: pairWithTokens }, + { ...historyEntry4, pair: pairWithTokens }, + ], ); - expect(mockMdwClient.getKeyBlockMicroBlocks).toHaveBeenCalledWith( - historyEntry4.height, + + mockMdwClient.getAccountBalanceForContractAtMicroBlockHash.mockImplementation( + ( + contractAddress: ContractAddress, + accountAddress: AccountAddress, + microBlockHash: MicroBlockHash, + ) => { + if (microBlockHash === historyEntry1.microBlockHash) { + if (contractAddress === pairWithTokens.token0.address) { + return { + amount: historyEntry1.reserve0, + }; + } else { + return { + amount: historyEntry1.reserve1, + }; + } + } else if (microBlockHash === historyEntry2.microBlockHash) { + if (contractAddress === pairWithTokens.token0.address) { + throw new Error('mdw error'); + } + } + }, ); - expect(mockMdwClient.getKeyBlockMicroBlocks).not.toHaveBeenCalledWith( - historyEntry5.height, + + mockPairLiquidityInfoHistoryDb.deleteFromMicroBlockTime.mockResolvedValue( + { count: 3 }, ); + + // Start validation + await service.validate(); + + // Assertions + expect( + mockMdwClient.getAccountBalanceForContractAtMicroBlockHash, + ).toHaveBeenCalledTimes(3); + expect( mockPairLiquidityInfoHistoryDb.deleteFromMicroBlockTime, - ).toHaveBeenCalledWith(historyEntry4.microBlockTime); - expect(service.logger.log).toHaveBeenCalledWith( - `Found an inconsistency in pair liquidity info history. Deleted 2 entries.`, - ); + ).toHaveBeenCalledWith(historyEntry2.microBlockTime); + expect(logSpy.mock.calls).toEqual([ + ['Started validating pair liquidity info history.'], + [ + 'Found an inconsistency in pair liquidity info history. Deleted 3 entries.', + ], + ['Finished validating pair liquidity info history.'], + ]); }); }); }); diff --git a/src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator.service.ts b/src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator.service.ts index e3a21c5..8658367 100644 --- a/src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator.service.ts +++ b/src/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator.service.ts @@ -1,54 +1,88 @@ import { Injectable, Logger } from '@nestjs/common'; -import { uniq } from 'lodash'; +import { groupBy, last, map } from 'lodash'; import { MdwHttpClientService } from '@/clients/mdw-http-client.service'; -import { MicroBlockHash } from '@/clients/sdk-client.model'; +import { + ContractAddress, + contractAddrToAccountAddr, + MicroBlockHash, +} from '@/clients/sdk-client.model'; import { SdkClientService } from '@/clients/sdk-client.service'; import { PairLiquidityInfoHistoryDbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-db.service'; +import { decimalToBigInt } from '@/lib/utils'; @Injectable() export class PairLiquidityInfoHistoryValidatorService { constructor( - private mdwClient: MdwHttpClientService, private pairLiquidityInfoHistoryDb: PairLiquidityInfoHistoryDbService, + private mdwClient: MdwHttpClientService, private sdkClient: SdkClientService, ) {} readonly logger = new Logger(PairLiquidityInfoHistoryValidatorService.name); + readonly VALIDATION_WINDOW_BLOCKS = 20; + async validate() { this.logger.log(`Started validating pair liquidity info history.`); // Get current height const currentHeight = await this.sdkClient.getHeight(); - // Get all liquidity entries greater or equal the current height minus 20 - const liquidityEntriesWithinHeightSorted = - await this.pairLiquidityInfoHistoryDb.getWithinHeightSorted( - currentHeight - 20, - ); - - // Get unique keyBlocks from entries - const uniqueHeights = uniq( - liquidityEntriesWithinHeightSorted.map((e) => e.height), + // Get all liquidity entries greater or equal the current height minus VALIDATION_WINDOW_BLOCKS + // and take the last entry of every microBlock to get the final reserve in that microBlock + const liquidityEntriesWithinHeightSorted = map( + groupBy( + await this.pairLiquidityInfoHistoryDb.getWithinHeightSortedWithPair( + currentHeight - this.VALIDATION_WINDOW_BLOCKS, + ), + 'microBlockHash', + ), + (group) => last(group)!, ); - // Fetch microBlocks for these unique keyBlocks from mdw - const microBlockHashsOnMdw = ( - await Promise.all( - uniqueHeights.map((h) => this.mdwClient.getKeyBlockMicroBlocks(h)), - ) - ) - .flat() - .map((mb) => mb.hash); - - // If a local microBlock is not contained in the mdw, delete this block and all newer entries + // If the reserves of a local microBlock do not match with the data from the middleware or the block does not exist, + // delete all logs in this block and all newer entries let numDeleted = 0; for (const liquidityEntry of liquidityEntriesWithinHeightSorted) { + let isError = false; + let mdwReserve0: bigint | undefined; + let mdwReserve1: bigint | undefined; + + try { + // reserve0 is the balance of the pair contract's account of token0 + mdwReserve0 = BigInt( + ( + await this.mdwClient.getAccountBalanceForContractAtMicroBlockHash( + liquidityEntry.pair.token0.address as ContractAddress, + contractAddrToAccountAddr( + liquidityEntry.pair.address as ContractAddress, + ), + liquidityEntry.microBlockHash as MicroBlockHash, + ) + ).amount, + ); + + // reserve1 is the balance of the pair contract's account of token1 + mdwReserve1 = BigInt( + ( + await this.mdwClient.getAccountBalanceForContractAtMicroBlockHash( + liquidityEntry.pair.token1.address as ContractAddress, + contractAddrToAccountAddr( + liquidityEntry.pair.address as ContractAddress, + ), + liquidityEntry.microBlockHash as MicroBlockHash, + ) + ).amount, + ); + } catch (e) { + this.logger.error(e); + isError = true; + } if ( - !microBlockHashsOnMdw.includes( - liquidityEntry.microBlockHash as MicroBlockHash, - ) + isError || + decimalToBigInt(liquidityEntry.reserve0) !== mdwReserve0 || + decimalToBigInt(liquidityEntry.reserve1) !== mdwReserve1 ) { numDeleted = ( await this.pairLiquidityInfoHistoryDb.deleteFromMicroBlockTime( diff --git a/src/tasks/tasks.module.ts b/src/tasks/tasks.module.ts index 137ae89..9365e32 100644 --- a/src/tasks/tasks.module.ts +++ b/src/tasks/tasks.module.ts @@ -4,9 +4,7 @@ import { ScheduleModule } from '@nestjs/schedule'; import { ClientsModule } from '@/clients/clients.module'; import { DatabaseModule } from '@/database/database.module'; import { PairLiquidityInfoHistoryImporterService } from '@/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service'; -import { PairLiquidityInfoHistoryImporterV2Service } from '@/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service'; import { PairLiquidityInfoHistoryValidatorService } from '@/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator.service'; -import { PairLiquidityInfoHistoryValidatorV2Service } from '@/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator-v2.service'; import { PairSyncService } from '@/tasks/pair-sync/pair-sync.service'; import { TasksService } from '@/tasks/tasks.service'; @@ -14,9 +12,7 @@ import { TasksService } from '@/tasks/tasks.service'; imports: [ClientsModule, DatabaseModule, ScheduleModule.forRoot()], providers: [ PairLiquidityInfoHistoryImporterService, - PairLiquidityInfoHistoryImporterV2Service, PairLiquidityInfoHistoryValidatorService, - PairLiquidityInfoHistoryValidatorV2Service, TasksService, PairSyncService, ], diff --git a/src/tasks/tasks.service.spec.ts b/src/tasks/tasks.service.spec.ts index b3dbec8..da422df 100644 --- a/src/tasks/tasks.service.spec.ts +++ b/src/tasks/tasks.service.spec.ts @@ -6,50 +6,42 @@ import { MdwHttpClientService } from '@/clients/mdw-http-client.service'; import { SdkClientService } from '@/clients/sdk-client.service'; import { PairDbService } from '@/database/pair/pair-db.service'; import { PairLiquidityInfoHistoryDbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-db.service'; -import { PairLiquidityInfoHistoryV2DbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-v2-db.service'; import { PairLiquidityInfoHistoryErrorDbService } from '@/database/pair-liquidity-info-history-error/pair-liquidity-info-history-error-db.service'; -import { PairLiquidityInfoHistoryV2ErrorDbService } from '@/database/pair-liquidity-info-history-error/pair-liquidity-info-history-v2-error-db.service'; import { PrismaService } from '@/database/prisma.service'; import { PairLiquidityInfoHistoryImporterService } from '@/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service'; -import { PairLiquidityInfoHistoryImporterV2Service } from '@/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service'; import { PairLiquidityInfoHistoryValidatorService } from '@/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator.service'; -import { PairLiquidityInfoHistoryValidatorV2Service } from '@/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator-v2.service'; import { TasksService } from '@/tasks/tasks.service'; describe('TasksService', () => { let tasksService: TasksService; - let pairLiquidityInfoHistoryImporterService: PairLiquidityInfoHistoryImporterV2Service; - let pairLiquidityInfoHistoryValidatorService: PairLiquidityInfoHistoryValidatorV2Service; + let pairLiquidityInfoHistoryImporterService: PairLiquidityInfoHistoryImporterService; + let pairLiquidityInfoHistoryValidatorService: PairLiquidityInfoHistoryValidatorService; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ providers: [ TasksService, PairLiquidityInfoHistoryImporterService, - PairLiquidityInfoHistoryImporterV2Service, PairLiquidityInfoHistoryValidatorService, - PairLiquidityInfoHistoryValidatorV2Service, HttpService, CoinmarketcapClientService, MdwHttpClientService, SdkClientService, PairDbService, PairLiquidityInfoHistoryDbService, - PairLiquidityInfoHistoryV2DbService, PairLiquidityInfoHistoryErrorDbService, - PairLiquidityInfoHistoryV2ErrorDbService, PrismaService, ], }).compile(); tasksService = module.get(TasksService); pairLiquidityInfoHistoryImporterService = - module.get( - PairLiquidityInfoHistoryImporterV2Service, + module.get( + PairLiquidityInfoHistoryImporterService, ); pairLiquidityInfoHistoryValidatorService = - module.get( - PairLiquidityInfoHistoryValidatorV2Service, + module.get( + PairLiquidityInfoHistoryValidatorService, ); }); @@ -59,7 +51,7 @@ describe('TasksService', () => { .spyOn(pairLiquidityInfoHistoryImporterService, 'import') .mockResolvedValue(); - await tasksService.runPairLiquidityInfoHistoryImporterV2(); + await tasksService.runPairLiquidityInfoHistoryImporter(); expect(pairLiquidityInfoHistoryImporterService.import).toHaveBeenCalled(); expect(tasksService.isRunning).toBe(false); }); @@ -67,7 +59,7 @@ describe('TasksService', () => { it('should not run if a task is running already', async () => { tasksService.setIsRunning(true); jest.spyOn(pairLiquidityInfoHistoryImporterService, 'import'); - await tasksService.runPairLiquidityInfoHistoryImporterV2(); + await tasksService.runPairLiquidityInfoHistoryImporter(); expect( pairLiquidityInfoHistoryImporterService.import, ).not.toHaveBeenCalled(); @@ -80,7 +72,7 @@ describe('TasksService', () => { .mockRejectedValue(error); jest.spyOn(pairLiquidityInfoHistoryImporterService.logger, 'error'); - await tasksService.runPairLiquidityInfoHistoryImporterV2(); + await tasksService.runPairLiquidityInfoHistoryImporter(); expect( pairLiquidityInfoHistoryImporterService.logger.error, ).toHaveBeenCalledWith(`Import failed. ${error}`); @@ -94,7 +86,7 @@ describe('TasksService', () => { .spyOn(pairLiquidityInfoHistoryValidatorService, 'validate') .mockResolvedValue(); - await tasksService.runPairLiquidityInfoHistoryValidatorV2(); + await tasksService.runPairLiquidityInfoHistoryValidator(); expect( pairLiquidityInfoHistoryValidatorService.validate, ).toHaveBeenCalled(); @@ -105,7 +97,7 @@ describe('TasksService', () => { jest.spyOn(pairLiquidityInfoHistoryValidatorService, 'validate'); - await tasksService.runPairLiquidityInfoHistoryValidatorV2(); + await tasksService.runPairLiquidityInfoHistoryValidator(); expect( pairLiquidityInfoHistoryValidatorService.validate, ).not.toHaveBeenCalled(); @@ -118,7 +110,7 @@ describe('TasksService', () => { .mockRejectedValue(error); jest.spyOn(pairLiquidityInfoHistoryValidatorService.logger, 'error'); - await tasksService.runPairLiquidityInfoHistoryValidatorV2(); + await tasksService.runPairLiquidityInfoHistoryValidator(); expect( pairLiquidityInfoHistoryValidatorService.logger.error, ).toHaveBeenCalledWith(`Validation failed. ${error}`); diff --git a/src/tasks/tasks.service.ts b/src/tasks/tasks.service.ts index 7a23693..56c071a 100644 --- a/src/tasks/tasks.service.ts +++ b/src/tasks/tasks.service.ts @@ -2,9 +2,7 @@ import { Injectable } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; import { PairLiquidityInfoHistoryImporterService } from '@/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service'; -import { PairLiquidityInfoHistoryImporterV2Service } from '@/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service'; import { PairLiquidityInfoHistoryValidatorService } from '@/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator.service'; -import { PairLiquidityInfoHistoryValidatorV2Service } from '@/tasks/pair-liquidity-info-history-validator/pair-liquidity-info-history-validator-v2.service'; const EVERY_5_MINUTES_STARTING_AT_02_30 = '30 2-57/5 * * * *'; @@ -12,9 +10,7 @@ const EVERY_5_MINUTES_STARTING_AT_02_30 = '30 2-57/5 * * * *'; export class TasksService { constructor( private pairLiquidityInfoHistoryImporterService: PairLiquidityInfoHistoryImporterService, - private pairLiquidityInfoHistoryImporterV2Service: PairLiquidityInfoHistoryImporterV2Service, private pairLiquidityInfoHistoryValidatorService: PairLiquidityInfoHistoryValidatorService, - private pairLiquidityInfoHistoryValidatorV2Service: PairLiquidityInfoHistoryValidatorV2Service, ) {} private _isRunning = false; @@ -27,64 +23,32 @@ export class TasksService { this._isRunning = isRunning; } - // @Cron(CronExpression.EVERY_5_MINUTES) - // async runPairLiquidityInfoHistoryImporter() { - // try { - // if (!this.isRunning) { - // this.setIsRunning(true); - // await this.pairLiquidityInfoHistoryImporterService.import(); - // this.setIsRunning(false); - // } - // } catch (error) { - // this.pairLiquidityInfoHistoryImporterService.logger.error( - // `Import failed. ${error}`, - // ); - // this.setIsRunning(false); - // } - // } - @Cron(CronExpression.EVERY_5_MINUTES) - async runPairLiquidityInfoHistoryImporterV2() { + async runPairLiquidityInfoHistoryImporter() { try { if (!this.isRunning) { this.setIsRunning(true); - await this.pairLiquidityInfoHistoryImporterV2Service.import(); + await this.pairLiquidityInfoHistoryImporterService.import(); this.setIsRunning(false); } } catch (error) { - this.pairLiquidityInfoHistoryImporterV2Service.logger.error( + this.pairLiquidityInfoHistoryImporterService.logger.error( `Import failed. ${error}`, ); this.setIsRunning(false); } } - // @Cron(EVERY_5_MINUTES_STARTING_AT_02_30) - // async runPairLiquidityInfoHistoryValidator() { - // try { - // if (!this.isRunning) { - // this.setIsRunning(true); - // await this.pairLiquidityInfoHistoryValidatorService.validate(); - // this.setIsRunning(false); - // } - // } catch (error) { - // this.pairLiquidityInfoHistoryValidatorService.logger.error( - // `Validation failed. ${error}`, - // ); - // this.setIsRunning(false); - // } - // } - @Cron(EVERY_5_MINUTES_STARTING_AT_02_30) - async runPairLiquidityInfoHistoryValidatorV2() { + async runPairLiquidityInfoHistoryValidator() { try { if (!this.isRunning) { this.setIsRunning(true); - await this.pairLiquidityInfoHistoryValidatorV2Service.validate(); + await this.pairLiquidityInfoHistoryValidatorService.validate(); this.setIsRunning(false); } } catch (error) { - this.pairLiquidityInfoHistoryValidatorV2Service.logger.error( + this.pairLiquidityInfoHistoryValidatorService.logger.error( `Validation failed. ${error}`, ); this.setIsRunning(false); diff --git a/test/e2e/__snapshots__/pair-liquidity-info-history-v2-db.service.e2e-spec.ts.snap b/test/e2e/__snapshots__/pair-liquidity-info-history-db.service.e2e-spec.ts.snap similarity index 79% rename from test/e2e/__snapshots__/pair-liquidity-info-history-v2-db.service.e2e-spec.ts.snap rename to test/e2e/__snapshots__/pair-liquidity-info-history-db.service.e2e-spec.ts.snap index 1d29cc4..e914ac4 100644 --- a/test/e2e/__snapshots__/pair-liquidity-info-history-v2-db.service.e2e-spec.ts.snap +++ b/test/e2e/__snapshots__/pair-liquidity-info-history-db.service.e2e-spec.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`PairLiquidityInfoHistoryV2DbService upsert should correctly upsert an existing entry 1`] = ` +exports[`PairLiquidityInfoHistoryDbService upsert should correctly upsert an existing entry 1`] = ` { "aeUsdPrice": "0.060559", "deltaReserve0": "-500", diff --git a/test/e2e/pair-liquidity-info-history-db.service.e2e-spec.ts b/test/e2e/pair-liquidity-info-history-db.service.e2e-spec.ts index 96bec5f..83247aa 100644 --- a/test/e2e/pair-liquidity-info-history-db.service.e2e-spec.ts +++ b/test/e2e/pair-liquidity-info-history-db.service.e2e-spec.ts @@ -1,106 +1,24 @@ import { Test, TestingModule } from '@nestjs/testing'; -import { Pair, PairLiquidityInfoHistory, Token } from '@prisma/client'; +import { Decimal } from '@prisma/client/runtime/library'; +import { omit } from 'lodash'; import { OrderQueryEnum } from '@/api/api.model'; import { ContractAddress } from '@/clients/sdk-client.model'; import { PairLiquidityInfoHistoryDbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-db.service'; import { PrismaService } from '@/database/prisma.service'; - -const token1: Token = { - id: 1, - address: 'ct_token1', - symbol: '1', - name: '1', - decimals: 18, - malformed: false, - noContract: false, - listed: false, -}; -const token2: Token = { - id: 2, - address: 'ct_token2', - symbol: '2', - name: '2', - decimals: 18, - malformed: false, - noContract: false, - listed: false, -}; -const token3: Token = { - id: 3, - address: 'ct_token3', - symbol: '3', - name: '3', - decimals: 18, - malformed: false, - noContract: false, - listed: false, -}; -const pair1: Pair = { - id: 1, - address: 'ct_pair1', - t0: 1, - t1: 2, - synchronized: true, -}; -const pair2: Pair = { - id: 2, - address: 'ct_pair2', - t0: 2, - t1: 3, - synchronized: true, -}; -const pair3: Pair = { - id: 3, - address: 'ct_pair4', - t0: 2, - t1: 3, - synchronized: true, -}; -const historyEntry1: PairLiquidityInfoHistory = { - id: 1, - pairId: 1, - totalSupply: '2000148656239820912122563', - reserve0: '950875688379385634428666', - reserve1: '4208476309359648851631167', - height: 100001, - microBlockHash: 'mh_entry1', - microBlockTime: 1000000000001n, - updatedAt: new Date(), -}; -const historyEntry2: PairLiquidityInfoHistory = { - id: 2, - pairId: 1, - totalSupply: '9954575303087659158151', - reserve0: '20210309618736130321327', - reserve1: '4903471477408475598460', - height: 200002, - microBlockHash: 'mh_entry2', - microBlockTime: 2000000000002n, - updatedAt: new Date(), -}; -const historyEntry3: PairLiquidityInfoHistory = { - id: 3, - pairId: 2, - totalSupply: '56931443813890767374824', - reserve0: '20556919390913460010617', - reserve1: '157691178959228289022449', - height: 300003, - microBlockHash: 'mh_entry3', - microBlockTime: 3000000000003n, - updatedAt: new Date(), -}; -const historyEntry4: PairLiquidityInfoHistory = { - id: 4, - pairId: 2, - totalSupply: '56931443813890767374824', - reserve0: '20556919390913460010617', - reserve1: '157691178959228289022449', - height: 300003, - microBlockHash: 'mh_entry4', - microBlockTime: 4000000000004n, - updatedAt: new Date(), -}; +import { EventType } from '@/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service'; +import { + historyEntry1, + historyEntry2, + historyEntry3, + historyEntry4, + pair1, + pair2, + pair3, + token1, + token2, + token3, +} from '@/test/mock-data/pair-liquidity-info-history-mock-data'; describe('PairLiquidityInfoHistoryDbService', () => { let service: PairLiquidityInfoHistoryDbService; @@ -121,7 +39,7 @@ describe('PairLiquidityInfoHistoryDbService', () => { await prismaService.token.createMany({ data: [token1, token2, token3] }); await prismaService.pair.createMany({ data: [pair1, pair2, pair3] }); await prismaService.pairLiquidityInfoHistory.createMany({ - data: [historyEntry1, historyEntry2, historyEntry3, historyEntry4], + data: [historyEntry4, historyEntry2, historyEntry3, historyEntry1], }); }); @@ -135,15 +53,100 @@ describe('PairLiquidityInfoHistoryDbService', () => { await prismaService.$disconnect(); }); + describe('upsert', () => { + it('should correctly upsert an existing entry', async () => { + const updatedEntry = { + pairId: historyEntry2.pairId, + eventType: EventType.PairBurn, + reserve0: new Decimal(500), + reserve1: new Decimal(500), + deltaReserve0: new Decimal(-500), + deltaReserve1: new Decimal(-500), + aeUsdPrice: new Decimal(0.060559), + height: 200002, + microBlockHash: historyEntry2.microBlockHash, + microBlockTime: 2000000000002n, + transactionHash: historyEntry2.transactionHash, + transactionIndex: 200002n, + logIndex: historyEntry2.logIndex, + }; + await service.upsert(updatedEntry); + const entry = await prismaService.pairLiquidityInfoHistory.findUnique({ + where: { id: historyEntry2.id }, + }); + expect(omit(entry, ['createdAt', 'updatedAt'])).toMatchSnapshot(); + }); + + it('should correctly insert an new entry', async () => { + const newEntry = { + pairId: 1, + eventType: EventType.PairBurn, + reserve0: new Decimal(500), + reserve1: new Decimal(500), + deltaReserve0: new Decimal(-500), + deltaReserve1: new Decimal(-500), + aeUsdPrice: new Decimal(0), + height: 500005, + microBlockHash: 'mh_entry5', + microBlockTime: 5000000000005n, + transactionHash: 'th_entry5', + transactionIndex: 500005n, + logIndex: 1, + }; + await service.upsert(newEntry); + const entry = await service.getLastlySyncedLogByPairId(1); + expect(entry?.microBlockHash).toEqual('mh_entry5'); + }); + }); + + describe('getLastlySyncedLogByPairId', () => { + it('should correctly return the last synced log for a given pairId', async () => { + const result1 = await service.getLastlySyncedLogByPairId(1); + const result2 = await service.getLastlySyncedLogByPairId(2); + expect(result1?.id).toEqual(historyEntry2.id); + expect(result2?.id).toEqual(historyEntry4.id); + }); + }); + + describe('getWithinHeightSortedWithPair', () => { + it('should correctly return all entries greater or equal a given height limit sorted ascending', async () => { + const result = await service.getWithinHeightSortedWithPair(200002); + expect(result.map((e) => e.id)).toEqual([ + historyEntry2.id, + historyEntry3.id, + historyEntry4.id, + ]); + }); + }); + + describe('deleteFromMicroBlockTime', () => { + it('should correctly delete all entries newer or equal a given block time', async () => { + await service.deleteFromMicroBlockTime(3000000000003n); + const result = await prismaService.pairLiquidityInfoHistory.findMany(); + expect(result.map((e) => e.id)).toEqual([ + historyEntry2.id, + historyEntry1.id, + ]); + }); + }); + describe('getAll', () => { it('should return all entries', async () => { const result = await service.getAll(100, 0, OrderQueryEnum.asc); - expect(result.map((e) => e.id)).toEqual([1, 2, 3, 4]); + expect(result.map((e) => e.id)).toEqual([ + historyEntry1.id, + historyEntry2.id, + historyEntry3.id, + historyEntry4.id, + ]); }); it('should return return entries with limit, offset and order', async () => { const result = await service.getAll(2, 1, OrderQueryEnum.desc); - expect(result.map((e) => e.id)).toEqual([3, 2]); + expect(result.map((e) => e.id)).toEqual([ + historyEntry3.id, + historyEntry2.id, + ]); }); it('should correctly filter by pair address', async () => { @@ -156,7 +159,10 @@ describe('PairLiquidityInfoHistoryDbService', () => { undefined, undefined, ); - expect(result.map((e) => e.id)).toEqual([1, 2]); + expect(result.map((e) => e.id)).toEqual([ + historyEntry1.id, + historyEntry2.id, + ]); }); it('should correctly filter by height', async () => { @@ -169,7 +175,10 @@ describe('PairLiquidityInfoHistoryDbService', () => { undefined, undefined, ); - expect(result.map((e) => e.id)).toEqual([3, 4]); + expect(result.map((e) => e.id)).toEqual([ + historyEntry3.id, + historyEntry4.id, + ]); }); it('should correctly return entries newer or equal fromBlockTime', async () => { @@ -182,7 +191,11 @@ describe('PairLiquidityInfoHistoryDbService', () => { 2000000000002n, undefined, ); - expect(result.map((e) => e.id)).toEqual([2, 3, 4]); + expect(result.map((e) => e.id)).toEqual([ + historyEntry2.id, + historyEntry3.id, + historyEntry4.id, + ]); }); it('should correctly return entries older or equal toBlockTime', async () => { @@ -195,29 +208,12 @@ describe('PairLiquidityInfoHistoryDbService', () => { undefined, 3000000000003n, ); - expect(result.map((e) => e.id)).toEqual([3, 2, 1]); - }); - }); - - describe('getLastlySyncedBlockByPairId', () => { - it('should correctly return the last synced block for a given pairId', async () => { - const result = await service.getLastlySyncedBlockByPairId(1); - expect(result?.microBlockTime).toEqual(2000000000002n); - }); - }); - - describe('getWithinHeightSorted', () => { - it('should correctly return all entries greater or equal a given height limit sorted by microBlockTime ascending', async () => { - const result = await service.getWithinHeightSorted(200002); - expect(result.map((e) => e.id)).toEqual([2, 3, 4]); - }); - }); - - describe('deleteFromMicroBlockTime', () => { - it('should correctly delete all entries newer or equal a given block time', async () => { - await service.deleteFromMicroBlockTime(3000000000003n); - const result = await prismaService.pairLiquidityInfoHistory.findMany(); - expect(result.map((e) => e.id)).toEqual([1, 2]); + expect(result.map((e) => e.id)).toEqual([ + historyEntry4.id, + historyEntry3.id, + historyEntry2.id, + historyEntry1.id, + ]); }); }); }); diff --git a/test/e2e/pair-liquidity-info-history-error-db.service.e2e-spec.ts b/test/e2e/pair-liquidity-info-history-error-db.service.e2e-spec.ts index ea240b9..2e3c23d 100644 --- a/test/e2e/pair-liquidity-info-history-error-db.service.e2e-spec.ts +++ b/test/e2e/pair-liquidity-info-history-error-db.service.e2e-spec.ts @@ -42,6 +42,8 @@ const errorEntry1: PairLiquidityInfoHistoryError = { id: 1, pairId: 1, microBlockHash: '', + transactionHash: '', + logIndex: -1, error: 'error_1', timesOccurred: 1, createdAt: new Date('2024-01-01 12:00:00.000'), @@ -51,6 +53,8 @@ const errorEntry2: PairLiquidityInfoHistoryError = { id: 2, pairId: 1, microBlockHash: 'mh_1', + transactionHash: 'th_1', + logIndex: 1, error: 'error_2', timesOccurred: 1, createdAt: new Date('2024-01-01 12:00:00.000'), @@ -92,32 +96,20 @@ describe('PairLiquidityInfoHistoryErrorDbService', () => { await prismaService.$disconnect(); }); - describe('getErrorByPairIdAndMicroBlockHashWithinHours', () => { - it('should correctly return an error within a recent given time window in hours by pairId', async () => { + describe('getErrorWithinHours', () => { + it('should correctly return an error within a recent given time window in hours on pair basis', async () => { jest.useFakeTimers().setSystemTime(new Date('2024-01-01 17:59:00.000')); - const result = await service.getErrorByPairIdAndMicroBlockHashWithinHours( - 1, - '', - 6, - ); + const result = await service.getErrorWithinHours(1, '', '', -1, 6); expect(result?.id).toEqual(1); }); - it('should correctly return an error within a recent given time window in hours by pairId and microBlockHash', async () => { - const result = await service.getErrorByPairIdAndMicroBlockHashWithinHours( - 1, - 'mh_1', - 6, - ); + it('should correctly return an error within a recent given time window in hours on log basis', async () => { + const result = await service.getErrorWithinHours(1, 'mh_1', 'th_1', 1, 6); expect(result?.id).toEqual(2); }); it('should not return errors older than the given time window in hours', async () => { - const result = await service.getErrorByPairIdAndMicroBlockHashWithinHours( - 1, - '', - 5, - ); + const result = await service.getErrorWithinHours(1, '', '', -1, 5); expect(result).toBe(null); }); }); diff --git a/test/e2e/pair-liquidity-info-history-v2-db.service.e2e-spec.ts b/test/e2e/pair-liquidity-info-history-v2-db.service.e2e-spec.ts deleted file mode 100644 index ed83b2f..0000000 --- a/test/e2e/pair-liquidity-info-history-v2-db.service.e2e-spec.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { Decimal } from '@prisma/client/runtime/library'; -import { omit } from 'lodash'; - -import { PairLiquidityInfoHistoryV2DbService } from '@/database/pair-liquidity-info-history/pair-liquidity-info-history-v2-db.service'; -import { PrismaService } from '@/database/prisma.service'; -import { EventType } from '@/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service'; -import { - historyEntry1, - historyEntry2, - historyEntry3, - historyEntry4, - pair1, - pair2, - pair3, - token1, - token2, - token3, -} from '@/test/mock-data/pair-liquidity-info-history-mock-data'; - -describe('PairLiquidityInfoHistoryV2DbService', () => { - let service: PairLiquidityInfoHistoryV2DbService; - let prismaService: PrismaService; - - beforeAll(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [PairLiquidityInfoHistoryV2DbService, PrismaService], - }).compile(); - - service = module.get( - PairLiquidityInfoHistoryV2DbService, - ); - prismaService = module.get(PrismaService); - }); - - beforeEach(async () => { - await prismaService.token.createMany({ data: [token1, token2, token3] }); - await prismaService.pair.createMany({ data: [pair1, pair2, pair3] }); - await prismaService.pairLiquidityInfoHistoryV2.createMany({ - data: [historyEntry4, historyEntry2, historyEntry3, historyEntry1], - }); - }); - - afterEach(async () => { - await prismaService.pairLiquidityInfoHistoryV2.deleteMany(); - await prismaService.pair.deleteMany(); - await prismaService.token.deleteMany(); - }); - - afterAll(async () => { - await prismaService.$disconnect(); - }); - - describe('upsert', () => { - it('should correctly upsert an existing entry', async () => { - const updatedEntry = { - pairId: historyEntry2.pairId, - eventType: EventType.PairBurn, - reserve0: new Decimal(500), - reserve1: new Decimal(500), - deltaReserve0: new Decimal(-500), - deltaReserve1: new Decimal(-500), - aeUsdPrice: new Decimal(0.060559), - height: 200002, - microBlockHash: historyEntry2.microBlockHash, - microBlockTime: 2000000000002n, - transactionHash: historyEntry2.transactionHash, - transactionIndex: 200002n, - logIndex: historyEntry2.logIndex, - }; - await service.upsert(updatedEntry); - const entry = await prismaService.pairLiquidityInfoHistoryV2.findUnique({ - where: { id: historyEntry2.id }, - }); - expect(omit(entry, ['createdAt', 'updatedAt'])).toMatchSnapshot(); - }); - - it('should correctly insert an new entry', async () => { - const newEntry = { - pairId: 1, - eventType: EventType.PairBurn, - reserve0: new Decimal(500), - reserve1: new Decimal(500), - deltaReserve0: new Decimal(-500), - deltaReserve1: new Decimal(-500), - aeUsdPrice: new Decimal(0), - height: 500005, - microBlockHash: 'mh_entry5', - microBlockTime: 5000000000005n, - transactionHash: 'th_entry5', - transactionIndex: 500005n, - logIndex: 1, - }; - await service.upsert(newEntry); - const entry = await service.getLastlySyncedLogByPairId(1); - expect(entry?.microBlockHash).toEqual('mh_entry5'); - }); - }); - - describe('getLastlySyncedLogByPairId', () => { - it('should correctly return the last synced log for a given pairId', async () => { - const result1 = await service.getLastlySyncedLogByPairId(1); - const result2 = await service.getLastlySyncedLogByPairId(2); - expect(result1?.id).toEqual(historyEntry2.id); - expect(result2?.id).toEqual(historyEntry4.id); - }); - }); - - describe('getWithinHeightSortedWithPair', () => { - it('should correctly return all entries greater or equal a given height limit sorted ascending', async () => { - const result = await service.getWithinHeightSortedWithPair(200002); - expect(result.map((e) => e.id)).toEqual([ - historyEntry2.id, - historyEntry3.id, - historyEntry4.id, - ]); - }); - }); - - describe('deleteFromMicroBlockTime', () => { - it('should correctly delete all entries newer or equal a given block time', async () => { - await service.deleteFromMicroBlockTime(3000000000003n); - const result = await prismaService.pairLiquidityInfoHistoryV2.findMany(); - expect(result.map((e) => e.id)).toEqual([ - historyEntry2.id, - historyEntry1.id, - ]); - }); - }); -}); diff --git a/test/e2e/pair-liquidity-info-history-v2-error-db.service.e2e-spec.ts b/test/e2e/pair-liquidity-info-history-v2-error-db.service.e2e-spec.ts deleted file mode 100644 index c261278..0000000 --- a/test/e2e/pair-liquidity-info-history-v2-error-db.service.e2e-spec.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { Pair, PairLiquidityInfoHistoryV2Error, Token } from '@prisma/client'; - -import { PairLiquidityInfoHistoryV2ErrorDbService } from '@/database/pair-liquidity-info-history-error/pair-liquidity-info-history-v2-error-db.service'; -import { PrismaService } from '@/database/prisma.service'; - -const token1: Token = { - id: 1, - address: 'ct_token1', - symbol: '1', - name: '1', - decimals: 18, - malformed: false, - noContract: false, - listed: false, -}; -const token2: Token = { - id: 2, - address: 'ct_token2', - symbol: '2', - name: '2', - decimals: 18, - malformed: false, - noContract: false, - listed: false, -}; -const pair1: Pair = { - id: 1, - address: 'ct_pair1', - t0: 1, - t1: 2, - synchronized: true, -}; -const pair2: Pair = { - id: 2, - address: 'ct_pair2', - t0: 1, - t1: 2, - synchronized: true, -}; -const errorEntry1: PairLiquidityInfoHistoryV2Error = { - id: 1, - pairId: 1, - microBlockHash: '', - transactionHash: '', - logIndex: -1, - error: 'error_1', - timesOccurred: 1, - createdAt: new Date('2024-01-01 12:00:00.000'), - updatedAt: new Date('2024-01-01 12:00:00.000'), -}; -const errorEntry2: PairLiquidityInfoHistoryV2Error = { - id: 2, - pairId: 1, - microBlockHash: 'mh_1', - transactionHash: 'th_1', - logIndex: 1, - error: 'error_2', - timesOccurred: 1, - createdAt: new Date('2024-01-01 12:00:00.000'), - updatedAt: new Date('2024-01-01 12:00:00.000'), -}; - -describe('PairLiquidityInfoHistoryV2ErrorDbService', () => { - let service: PairLiquidityInfoHistoryV2ErrorDbService; - let prismaService: PrismaService; - - beforeAll(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [PairLiquidityInfoHistoryV2ErrorDbService, PrismaService], - }).compile(); - - service = module.get( - PairLiquidityInfoHistoryV2ErrorDbService, - ); - prismaService = module.get(PrismaService); - }); - - beforeEach(async () => { - await prismaService.token.createMany({ data: [token1, token2] }); - await prismaService.pair.createMany({ data: [pair1, pair2] }); - await prismaService.pairLiquidityInfoHistoryV2Error.createMany({ - data: [errorEntry1, errorEntry2], - }); - jest.useFakeTimers().setSystemTime(new Date('2024-01-01 17:59:00.000')); - }); - - afterEach(async () => { - await prismaService.pairLiquidityInfoHistoryV2Error.deleteMany(); - await prismaService.pair.deleteMany(); - await prismaService.token.deleteMany(); - jest.useRealTimers(); - }); - - afterAll(async () => { - await prismaService.$disconnect(); - }); - - describe('getErrorWithinHours', () => { - it('should correctly return an error within a recent given time window in hours on pair basis', async () => { - jest.useFakeTimers().setSystemTime(new Date('2024-01-01 17:59:00.000')); - const result = await service.getErrorWithinHours(1, '', '', -1, 6); - expect(result?.id).toEqual(1); - }); - - it('should correctly return an error within a recent given time window in hours on log basis', async () => { - const result = await service.getErrorWithinHours(1, 'mh_1', 'th_1', 1, 6); - expect(result?.id).toEqual(2); - }); - - it('should not return errors older than the given time window in hours', async () => { - const result = await service.getErrorWithinHours(1, '', '', -1, 5); - expect(result).toBe(null); - }); - }); -}); diff --git a/test/mock-data/pair-liquidity-info-history-mock-data.ts b/test/mock-data/pair-liquidity-info-history-mock-data.ts index 5103469..91d2aa2 100644 --- a/test/mock-data/pair-liquidity-info-history-mock-data.ts +++ b/test/mock-data/pair-liquidity-info-history-mock-data.ts @@ -1,9 +1,9 @@ -import { Pair, PairLiquidityInfoHistoryV2, Token } from '@prisma/client'; +import { Pair, PairLiquidityInfoHistory, Token } from '@prisma/client'; import { Decimal } from '@prisma/client/runtime/library'; import { Contract } from '@/clients/mdw-http-client.model'; import { ContractAddress } from '@/clients/sdk-client.model'; -import { EventType } from '@/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer-v2.service'; +import { EventType } from '@/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service'; export const token1: Token = { id: 1, @@ -62,7 +62,7 @@ export const pair3: Pair = { synchronized: true, }; -export const historyEntry1: PairLiquidityInfoHistoryV2 = { +export const historyEntry1: PairLiquidityInfoHistory = { id: 111, pairId: 1, eventType: EventType.PairMint, @@ -81,7 +81,7 @@ export const historyEntry1: PairLiquidityInfoHistoryV2 = { updatedAt: new Date(), }; -export const historyEntry2: PairLiquidityInfoHistoryV2 = { +export const historyEntry2: PairLiquidityInfoHistory = { id: 222, pairId: 1, eventType: EventType.SwapTokens, @@ -100,7 +100,7 @@ export const historyEntry2: PairLiquidityInfoHistoryV2 = { updatedAt: new Date(), }; -export const historyEntry3: PairLiquidityInfoHistoryV2 = { +export const historyEntry3: PairLiquidityInfoHistory = { id: 333, pairId: 2, eventType: EventType.PairMint, @@ -119,7 +119,7 @@ export const historyEntry3: PairLiquidityInfoHistoryV2 = { updatedAt: new Date(), }; -export const historyEntry4: PairLiquidityInfoHistoryV2 = { +export const historyEntry4: PairLiquidityInfoHistory = { id: 444, pairId: 2, eventType: EventType.SwapTokens, From 54240fa622adbcc20c7833d44dc472902a0e8010 Mon Sep 17 00:00:00 2001 From: Timo Erdelt Date: Mon, 6 May 2024 15:13:19 +0200 Subject: [PATCH 09/10] test: feedback: use it.skip() for commented out test --- ...-liquidity-info-history.controller.spec.ts | 151 ++++++++---------- 1 file changed, 64 insertions(+), 87 deletions(-) diff --git a/src/api/pair-liquidity-info-history/pair-liquidity-info-history.controller.spec.ts b/src/api/pair-liquidity-info-history/pair-liquidity-info-history.controller.spec.ts index 83b02a9..806efb1 100644 --- a/src/api/pair-liquidity-info-history/pair-liquidity-info-history.controller.spec.ts +++ b/src/api/pair-liquidity-info-history/pair-liquidity-info-history.controller.spec.ts @@ -1,10 +1,17 @@ import { INestApplication } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; +import { Pair, PairLiquidityInfoHistory } from '@prisma/client'; import * as request from 'supertest'; import { OrderQueryEnum } from '@/api/api.model'; import { PairLiquidityInfoHistoryController } from '@/api/pair-liquidity-info-history/pair-liquidity-info-history.controller'; import { PairLiquidityInfoHistoryService } from '@/api/pair-liquidity-info-history/pair-liquidity-info-history.service'; +import { + historyEntry1, + historyEntry3, + pair1, + pair2, +} from '@/test/mock-data/pair-liquidity-info-history-mock-data'; const mockPairLiquidityInfoHistoryService = { getAllHistoryEntries: jest.fn(), @@ -34,93 +41,63 @@ describe('PairLiquidityInfoHistoryController', () => { }); // TODO fix with updated return - // it('should return history entries and use default values for empty params', async () => { - // // Mocks - // const historyEntry1: { pair: Pair } & PairLiquidityInfoHistory = { - // id: 1, - // pairId: 1, - // totalSupply: '2000148656239820912122563', - // reserve0: '950875688379385634428666', - // reserve1: '4208476309359648851631167', - // height: 912485, - // microBlockHash: 'mh_Tx43Gh3acudUNSUWihPcV1Se4XcoFK3aUFAtFZk2Z4Zv7igZs', - // microBlockTime: 1709027642807n, - // updatedAt: new Date('2024-03-20 17:04:51.625'), - // pair: { - // id: 1, - // address: 'ct_efYtiwDg4YZxDWE3iLPzvrjb92CJPvzGwriv4ZRuvuTDMNMb9', - // t0: 15, - // t1: 5, - // synchronized: true, - // }, - // }; - // - // const historyEntry2: { pair: Pair } & PairLiquidityInfoHistory = { - // id: 2, - // pairId: 3, - // totalSupply: '9954575303087659158151', - // reserve0: '20210309618736130321327', - // reserve1: '4903471477408475598460', - // height: 707395, - // microBlockHash: 'mh_2dUTfmwFc2ymeroB534giVwEvsa8d44Vf8SXtvy6GeHjdgQoHj', - // microBlockTime: 1671708830503n, - // updatedAt: new Date('2024-03-20 12:16:49.065'), - // pair: { - // id: 3, - // address: 'ct_22iY9F7hng23gN8awi4aGnLy54YSR41wztbqgQCquuLYvTiGcm', - // t0: 17, - // t1: 22, - // synchronized: true, - // }, - // }; - // - // mockPairLiquidityInfoHistoryService.getAllHistoryEntries.mockResolvedValue( - // [historyEntry1, historyEntry2], - // ); - // - // // Call route - // const result = await request(app.getHttpServer()).get( - // '/history/liquidity', - // ); - // - // // Assertions - // expect( - // mockPairLiquidityInfoHistoryService.getAllHistoryEntries, - // ).toHaveBeenCalledWith( - // 100, - // 0, - // OrderQueryEnum.asc, - // undefined, - // undefined, - // undefined, - // undefined, - // ); - // expect(result.status).toBe(200); - // expect(result.body).toEqual([ - // { - // pairAddress: historyEntry1.pair.address, - // liquidityInfo: { - // totalSupply: historyEntry1.totalSupply, - // reserve0: historyEntry1.reserve0, - // reserve1: historyEntry1.reserve1, - // }, - // height: historyEntry1.height, - // microBlockHash: historyEntry1.microBlockHash, - // microBlockTime: historyEntry1.microBlockTime.toString(), - // }, - // { - // pairAddress: historyEntry2.pair.address, - // liquidityInfo: { - // totalSupply: historyEntry2.totalSupply, - // reserve0: historyEntry2.reserve0, - // reserve1: historyEntry2.reserve1, - // }, - // height: historyEntry2.height, - // microBlockHash: historyEntry2.microBlockHash, - // microBlockTime: historyEntry2.microBlockTime.toString(), - // }, - // ]); - // }); + it.skip('should return history entries and use default values for empty params', async () => { + // Mocks + const historyEntryWithPair1: { pair: Pair } & PairLiquidityInfoHistory = { + ...historyEntry1, + pair: pair1, + }; + + const historyEntryWithPair2: { pair: Pair } & PairLiquidityInfoHistory = { + ...historyEntry3, + pair: pair2, + }; + + mockPairLiquidityInfoHistoryService.getAllHistoryEntries.mockResolvedValue( + [historyEntryWithPair1, historyEntryWithPair2], + ); + + // Call route + const result = await request(app.getHttpServer()).get( + '/history/liquidity', + ); + + // Assertions + expect( + mockPairLiquidityInfoHistoryService.getAllHistoryEntries, + ).toHaveBeenCalledWith( + 100, + 0, + OrderQueryEnum.asc, + undefined, + undefined, + undefined, + undefined, + ); + expect(result.status).toBe(200); + expect(result.body).toEqual([ + { + pairAddress: historyEntryWithPair1.pair.address, + liquidityInfo: { + reserve0: historyEntryWithPair1.reserve0, + reserve1: historyEntryWithPair1.reserve1, + }, + height: historyEntryWithPair1.height, + microBlockHash: historyEntryWithPair1.microBlockHash, + microBlockTime: historyEntryWithPair1.microBlockTime.toString(), + }, + { + pairAddress: historyEntryWithPair2.pair.address, + liquidityInfo: { + reserve0: historyEntryWithPair2.reserve0, + reserve1: historyEntryWithPair2.reserve1, + }, + height: historyEntryWithPair2.height, + microBlockHash: historyEntryWithPair2.microBlockHash, + microBlockTime: historyEntryWithPair2.microBlockTime.toString(), + }, + ]); + }); it('should parse all query params correctly', async () => { // Mocks From fe8331b8be279d8dd6a24a4796ca095182a6e306 Mon Sep 17 00:00:00 2001 From: Timo Erdelt Date: Mon, 6 May 2024 16:09:14 +0200 Subject: [PATCH 10/10] test: include coinmarketcap API error in importer unit test --- ...info-history-importer.service.spec.ts.snap | 2 +- ...dity-info-history-importer.service.spec.ts | 26 ++++++++++--------- ...liquidity-info-history-importer.service.ts | 4 --- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer.service.spec.ts.snap b/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer.service.spec.ts.snap index ac26fb2..d34b712 100644 --- a/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer.service.spec.ts.snap +++ b/src/tasks/pair-liquidity-info-history-importer/__snapshots__/pair-liquidity-info-history-importer.service.spec.ts.snap @@ -112,7 +112,7 @@ exports[`PairLiquidityInfoHistoryImporterService import should skip a log if the ] `; -exports[`PairLiquidityInfoHistoryImporterService import should catch and insert an error on log level 1`] = ` +exports[`PairLiquidityInfoHistoryImporterService import should catch and insert any error on log level (e.g. API rate limit reached) 1`] = ` [ [ { diff --git a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.spec.ts b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.spec.ts index abcccdc..21b8a4a 100644 --- a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.spec.ts +++ b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.spec.ts @@ -268,13 +268,15 @@ describe('PairLiquidityInfoHistoryImporterService', () => { ); }); - it('should catch and insert an error on log level', async () => { + it('should catch and insert any error on log level (e.g. API rate limit reached)', async () => { + const errorMessage = + "You've exceeded your API Key's HTTP request rate limit. Rate limits reset every minute."; const error = { pairId: pairWithTokens.id, microBlockHash: contractLog3.block_hash, transactionHash: contractLog3.call_tx_hash, logIndex: parseInt(contractLog3.log_idx), - error: 'Error: error', + error: `Error: ${errorMessage}`, }; // Mock functions @@ -282,26 +284,26 @@ describe('PairLiquidityInfoHistoryImporterService', () => { mockPairLiquidityInfoHistoryErrorDb.getErrorWithinHours.mockResolvedValue( undefined, ); - mockPairLiquidityInfoHistoryDb.getLastlySyncedLogByPairId - .mockResolvedValueOnce({}) - .mockRejectedValueOnce(new Error('error')) - .mockRejectedValueOnce(undefined); mockMdwClient.getContractLogsUntilCondition.mockResolvedValue([ contractLog3, contractLog4, contractLog5, ]); - mockCoinmarketcapClient.getHistoricalPriceDataThrottled.mockResolvedValue( - coinmarketcapResponseAeUsdQuoteData, - ); - mockPairLiquidityInfoHistoryDb.upsert.mockResolvedValue(null); - mockPairLiquidityInfoHistoryErrorDb.upsert.mockResolvedValue(null); + mockPairLiquidityInfoHistoryDb.getLastlySyncedLogByPairId + .mockResolvedValueOnce({}) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined); + mockCoinmarketcapClient.getHistoricalPriceDataThrottled + .mockRejectedValueOnce(new Error(errorMessage)) + .mockResolvedValueOnce(coinmarketcapResponseAeUsdQuoteData); + + mockPairLiquidityInfoHistoryDb.upsert.mockResolvedValueOnce(null); + mockPairLiquidityInfoHistoryErrorDb.upsert.mockResolvedValueOnce(null); // Start import await service.import(); // Assertions - expect(errorSpy.mock.calls).toEqual([ [`Skipped log. ${JSON.stringify(error)}`], ]); diff --git a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.ts b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.ts index 6d2dc8a..d975769 100644 --- a/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.ts +++ b/src/tasks/pair-liquidity-info-history-importer/pair-liquidity-info-history-importer.service.ts @@ -359,9 +359,5 @@ export class PairLiquidityInfoHistoryImporterService { return this.coinmarketcapClient .getHistoricalPriceDataThrottled(microBlockTime) .then((res) => res.data['1700'].quotes[0].quote.USD.price); - // .catch((err) => { - // console.log(err); - // return 0; - // }); } }