From 551efc3f8856c5a6b5bdaa5f2f43b86c0ab818f1 Mon Sep 17 00:00:00 2001 From: ducphamle2 Date: Tue, 25 Jul 2023 09:28:21 +0700 Subject: [PATCH 1/5] pumped cosmos rpc sync to fix duplication sync --- packages/oraidex-server/package.json | 1 - packages/oraidex-sync/package.json | 2 +- yarn.lock | 15 ++++----------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/packages/oraidex-server/package.json b/packages/oraidex-server/package.json index f459715a..f8388623 100644 --- a/packages/oraidex-server/package.json +++ b/packages/oraidex-server/package.json @@ -14,7 +14,6 @@ "@cosmjs/stargate": "^0.31.0", "@oraichain/common-contracts-sdk": "1.0.13", "@cosmjs/tendermint-rpc": "^0.31.0", - "@oraichain/cosmos-rpc-sync": "^1.0.5", "@oraichain/oraidex-sync": "1.0.0", "cors": "^2.8.5", "express": "^4.18.2" diff --git a/packages/oraidex-sync/package.json b/packages/oraidex-sync/package.json index bb34cef0..5e9579b4 100644 --- a/packages/oraidex-sync/package.json +++ b/packages/oraidex-sync/package.json @@ -12,7 +12,7 @@ "dependencies": { "@cosmjs/tendermint-rpc": "^0.31.0", "@oraichain/common-contracts-sdk": "1.0.13", - "@oraichain/cosmos-rpc-sync": "^1.0.6", + "@oraichain/cosmos-rpc-sync": "^1.0.7", "@oraichain/oraidex-contracts-sdk": "^1.0.13", "duckdb-async": "^0.8.1", "apache-arrow": "^12.0.1" diff --git a/yarn.lock b/yarn.lock index a2dde450..eae19908 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1945,17 +1945,10 @@ resolved "https://registry.yarnpkg.com/@oraichain/common-contracts-sdk/-/common-contracts-sdk-1.0.13.tgz#5a6cd9320b995e184b5d484313e4b4e8ecd4b4bb" integrity sha512-XfDDaggu7WcM/vxRxIn0ipLq6+c9+5FBqK/qzWh5HRHxn4e71OvNKVAxXoLOIATscAXkrkOimv55s1CD+hZGmw== -"@oraichain/cosmos-rpc-sync@^1.0.5": - version "1.0.5" - resolved "https://registry.yarnpkg.com/@oraichain/cosmos-rpc-sync/-/cosmos-rpc-sync-1.0.5.tgz#252e2434c3fe1a2e82793cf42b842e9d939e3aa1" - integrity sha512-8QTlXEE2p/S2jH3tsVYBX+7zCPvNgOdfkz3CMMkNOKaTtiZc+9Q9SvuEb4ED2exhuD29MVzIMLpqjRicIHTEvA== - dependencies: - "@cosmjs/stargate" "^0.31.0" - -"@oraichain/cosmos-rpc-sync@^1.0.6": - version "1.0.6" - resolved "https://registry.yarnpkg.com/@oraichain/cosmos-rpc-sync/-/cosmos-rpc-sync-1.0.6.tgz#bdf59afce6d4308a3f7fec4723d40052b5ac9e7e" - integrity sha512-3LJgKT+lpTdThbpw6W1oZvgu3oAkep8SJ+qbE4tPYVaN4rD4SAx3IV0z8ccBPXgVs2t3yjaHfx9y++TQs7ZN0Q== +"@oraichain/cosmos-rpc-sync@^1.0.7": + version "1.0.7" + resolved "https://registry.yarnpkg.com/@oraichain/cosmos-rpc-sync/-/cosmos-rpc-sync-1.0.7.tgz#9113f2d828490080e2e8d88180428f7dcf69c0e8" + integrity sha512-2m+2wzjX94AwSm43xv70UL7TBPJmiS2BEw7WNN9CBw+OG93oiFAP0vXjF/fG/LrsuGHoLwXA9qQEDs5YprZ4/A== dependencies: "@cosmjs/stargate" "^0.31.0" From 9adb93d28dd9907b4c61621136351a855998cc81 Mon Sep 17 00:00:00 2001 From: ducphamle2 Date: Thu, 27 Jul 2023 22:45:28 +0700 Subject: [PATCH 2/5] added ohlcv for swap ops --- packages/contracts-build/package.json | 2 +- packages/oraidex-sync/package.json | 3 +- packages/oraidex-sync/src/db.ts | 32 ++++-- packages/oraidex-sync/src/helper.ts | 110 ++++++++++++++++---- packages/oraidex-sync/src/index.ts | 73 ++------------ packages/oraidex-sync/src/pairs.ts | 6 ++ packages/oraidex-sync/src/tx-parsing.ts | 11 +- packages/oraidex-sync/src/types.ts | 32 +++--- packages/oraidex-sync/tests/db.spec.ts | 8 ++ packages/oraidex-sync/tests/helper.spec.ts | 111 ++++++++++++++++----- yarn.lock | 5 + 11 files changed, 252 insertions(+), 141 deletions(-) diff --git a/packages/contracts-build/package.json b/packages/contracts-build/package.json index 2496a825..ffa31b13 100644 --- a/packages/contracts-build/package.json +++ b/packages/contracts-build/package.json @@ -8,6 +8,6 @@ "data/" ], "dependencies": { - "@oraichain/oraidex-contracts-sdk": "^1.0.10" + "@oraichain/oraidex-contracts-sdk": "^1.0.18" } } diff --git a/packages/oraidex-sync/package.json b/packages/oraidex-sync/package.json index 5e9579b4..29ff7171 100644 --- a/packages/oraidex-sync/package.json +++ b/packages/oraidex-sync/package.json @@ -15,7 +15,8 @@ "@oraichain/cosmos-rpc-sync": "^1.0.7", "@oraichain/oraidex-contracts-sdk": "^1.0.13", "duckdb-async": "^0.8.1", - "apache-arrow": "^12.0.1" + "apache-arrow": "^12.0.1", + "lodash": "^4.17.21" }, "devDependencies": { "@types/lodash": "^4.14.195" diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index ee30d8d5..90561496 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -1,12 +1,12 @@ import { Database, Connection } from "duckdb-async"; import { + Ohlcv, PairInfoData, PriceInfo, SwapOperationData, TokenVolumeData, TotalLiquidity, VolumeData, - VolumeInfo, WithdrawLiquidityOperationData } from "./types"; import fs, { rename } from "fs"; @@ -54,10 +54,17 @@ export class DuckDb { // swap operation table handling async createSwapOpsTable() { + try { + await this.conn.all("select enum_range(NULL::directionType);"); + } catch (error) { + // if error it means the enum does not exist => create new + await this.conn.exec("CREATE TYPE directionType AS ENUM ('Buy','Sell');"); + } await this.conn.exec( `CREATE TABLE IF NOT EXISTS swap_ops_data ( askDenom VARCHAR, commissionAmount UBIGINT, + direction directionType, offerAmount UBIGINT, offerDenom VARCHAR, uniqueKey VARCHAR UNIQUE, @@ -278,26 +285,29 @@ export class DuckDb { return result as TotalLiquidity[]; } - async createVolumeInfo() { + async createSwapOhlcv() { await this.conn.exec( - `CREATE TABLE IF NOT EXISTS volume_info ( - denom VARCHAR, - timestamp UINTEGER, - txheight UINTEGER, - price DOUBLE, - volume UBIGINT) + `CREATE TABLE IF NOT EXISTS swap_ohlcv ( + timestamp uinteger, + pair varchar, + price double, + volume ubigint, + open double, + close double, + low double, + high double) ` ); } - async insertVolumeInfo(volumeInfos: VolumeInfo[]) { - await this.insertBulkData(volumeInfos, "volume_info"); + async insertOhlcv(ohlcv: Ohlcv[]) { + await this.insertBulkData(ohlcv, "swap_ohlcv"); } async pivotVolumeRange(startTime: number, endTime: number) { let volumeInfos = await this.conn.all( ` - pivot (select * from volume_info + pivot (select * from swap_ohlcv where timestamp >= ${startTime} and timestamp <= ${endTime} order by timestamp) diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index 2bb84438..eeb1e510 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -1,15 +1,18 @@ import { AssetInfo, SwapOperation } from "@oraichain/oraidex-contracts-sdk"; -import { pairs } from "./pairs"; +import { pairs, pairsOnlyDenom } from "./pairs"; import { ORAI, atomic, tenAmountInDecimalSix, truncDecimals, usdtCw20Address } from "./constants"; import { + Ohlcv, OraiDexType, PairInfoData, - PrefixSumHandlingData, ProvideLiquidityOperationData, + SwapDirection, SwapOperationData, + TradeItem, WithdrawLiquidityOperationData } from "./types"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; +import { minBy, maxBy } from "lodash"; export function toObject(data: any[]) { return JSON.parse( @@ -107,20 +110,6 @@ async function delay(timeout: number) { return new Promise((resolve) => setTimeout(resolve, timeout)); } -function calculatePrefixSum(initialAmount: number, handlingData: PrefixSumHandlingData[]): PrefixSumHandlingData[] { - let prefixSumObj = {}; - for (let data of handlingData) { - if (!(`temp-${data.denom}` in prefixSumObj)) { - prefixSumObj[`temp-${data.denom}`] = initialAmount + data.amount; - data.amount = prefixSumObj[`temp-${data.denom}`]; - continue; - } - prefixSumObj[`temp-${data.denom}`] += data.amount; - data.amount = prefixSumObj[`temp-${data.denom}`]; - } - return handlingData; -} - function findMappedTargetedAssetInfo(targetedAssetInfo: AssetInfo): AssetInfo[] { const mappedAssetInfos = []; @@ -175,7 +164,7 @@ function calculatePriceByPool(offerPool: bigint, askPool: bigint, commissionRate export function groupByTime(data: any[], timeframe?: number): any[] { let ops: { [k: number]: any[] } = {}; for (const op of data) { - const roundedTime = roundTime(op.timestamp * 1000, timeframe || 60); + const roundedTime = roundTime(op.timestamp * 1000, timeframe || 60); // op timestamp is sec if (!ops[roundedTime]) { ops[roundedTime] = []; } @@ -225,8 +214,9 @@ export function collectAccumulateLpData( ); if (!pool) continue; if (op.opType === "withdraw") { - op.firstTokenLp = BigInt(op.firstTokenLp) - BigInt(op.firstTokenLp) * 2n; - op.secondTokenLp = BigInt(op.secondTokenLp) - BigInt(op.secondTokenLp) * 2n; + // reverse sign since withdraw means lp decreases + op.firstTokenLp = -BigInt(op.firstTokenLp); + op.secondTokenLp = -BigInt(op.secondTokenLp); } const denom = `${op.firstTokenDenom} - ${op.secondTokenDenom}`; if (!accumulateData[denom]) { @@ -249,8 +239,6 @@ export function collectAccumulateLpData( op.firstTokenLp = accumulateData[denom].firstTokenAmount; op.secondTokenLp = accumulateData[denom].secondTokenAmount; } - - // convert bigint to number so we can store them into the db without error } export function removeOpsDuplication(ops: OraiDexType[]): OraiDexType[] { @@ -261,6 +249,85 @@ export function removeOpsDuplication(ops: OraiDexType[]): OraiDexType[] { return newOps; } +/** + * Group swapOps have same pair. + * @param swapOps + * @returns + */ +export function groupSwapOpsByPair(ops: SwapOperationData[]): { [key: string]: SwapOperationData[] } { + let opsByPair = {}; + for (const op of ops) { + const pairIndex = findPairIndexFromDenoms(op.offerDenom, op.askDenom); + if (pairIndex === -1) continue; + const pair = JSON.stringify(pairs[pairIndex].asset_infos); + if (!opsByPair[pair]) { + opsByPair[pair] = []; + } + opsByPair[pair].push(op); + } + return opsByPair; +} + +export function calculateOhlcv(orders: TradeItem[]): Ohlcv { + const timestamp = orders[0].timestamp; + const pair = orders[0].pair; + const open = orders[0].price; + const close = orders[orders.length - 1].price; + const low = minBy(orders, "price").price; + const high = maxBy(orders, "price").price; + const volume = orders.reduce((acc, currentValue) => { + return acc + currentValue.volume; + }, BigInt(0)); + + return { + open, + close, + low, + high, + volume, + timestamp, + pair + }; +} + +export function buildOhlcv(ops: SwapOperationData[]): Ohlcv[] { + let ohlcv: Ohlcv[] = []; + for (const [_, opsByPair] of Object.entries(groupSwapOpsByPair(ops))) { + const orderByTimes = groupByTime(opsByPair); + const ticks = Object.values(orderByTimes).map((value) => calculateOhlcv(value)); + ohlcv.push(...ticks); + } + return ohlcv; +} + +export function calculatePriceFromSwapOp(op: SwapOperationData): number { + if (!op || !op.offerAmount || !op.returnAmount) { + return 0; + } + const offerAmount = op.offerAmount; + const askAmount = op.returnAmount; + return op.direction === "Buy" ? Number(offerAmount) / Number(askAmount) : Number(askAmount) / Number(offerAmount); +} + +export function getSwapDirection(offerDenom: string, askDenom: string): SwapDirection { + const pair = pairsOnlyDenom.find( + (pair) => pair.asset_infos.some((info) => info === offerDenom) && pair.asset_infos.some((info) => info === askDenom) + ); + if (!pair) { + throw Error("Cannot find asset infos in list of pairs"); + } + const assetInfos = pair.asset_infos; + // use quote denom as offer then its buy. Quote denom in pairs is the 2nd index in the array + if (assetInfos[0] === offerDenom) return "Sell"; + return "Buy"; +} + +export function findPairIndexFromDenoms(offerDenom: string, askDenom: string): number { + return pairsOnlyDenom.findIndex( + (pair) => pair.asset_infos.some((info) => info === offerDenom) && pair.asset_infos.some((info) => info === askDenom) + ); +} + // /** // * // * @param infos @@ -283,7 +350,6 @@ export function removeOpsDuplication(ops: OraiDexType[]): OraiDexType[] { // } export { - calculatePrefixSum, findMappedTargetedAssetInfo, findAssetInfoPathToUsdt, generateSwapOperations, diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index 35fc5e7a..5c059e11 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -2,30 +2,19 @@ import "dotenv/config"; import { parseAssetInfo, parseTxs } from "./tx-parsing"; import { DuckDb } from "./db"; import { WriteData, SyncData, Txs } from "@oraichain/cosmos-rpc-sync"; -import { pairs } from "./pairs"; -import { - Asset, - AssetInfo, - CosmWasmClient, - OraiswapFactoryQueryClient, - OraiswapRouterQueryClient, - PairInfo -} from "@oraichain/oraidex-contracts-sdk"; +import { CosmWasmClient, OraiswapFactoryQueryClient, PairInfo } from "@oraichain/oraidex-contracts-sdk"; import { ProvideLiquidityOperationData, - SwapOperationData, TxAnlysisResult, WithdrawLiquidityOperationData, InitialData, PairInfoData, - Env, - VolumeInfo, - PrefixSumHandlingData + Env } from "./types"; import { MulticallQueryClient } from "@oraichain/common-contracts-sdk"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; -import { getAllPairInfos, getPoolInfos, simulateSwapPriceWithUsdt } from "./query"; -import { calculatePrefixSum, collectAccumulateLpData, parseAssetInfoOnlyDenom } from "./helper"; +import { getAllPairInfos, getPoolInfos } from "./query"; +import { collectAccumulateLpData } from "./helper"; class WriteOrders extends WriteData { private firstWrite: boolean; @@ -34,27 +23,13 @@ class WriteOrders extends WriteData { this.firstWrite = true; } - private async insertSwapOps(ops: SwapOperationData[]) { - await this.duckDb.insertSwapOps(ops); - } - - private async insertLiquidityOps(ops: ProvideLiquidityOperationData[] | WithdrawLiquidityOperationData[]) { - await this.duckDb.insertLpOps(ops); - } - private async insertParsedTxs(txs: TxAnlysisResult) { // insert swap ops await Promise.all([ - this.insertSwapOps(txs.swapOpsData), - this.insertLiquidityOps(txs.provideLiquidityOpsData), - this.duckDb.insertVolumeInfo(txs.volumeInfos) + this.duckDb.insertSwapOps(txs.swapOpsData), + this.duckDb.insertLpOps([...txs.provideLiquidityOpsData, ...txs.withdrawLiquidityOpsData]), + this.duckDb.insertOhlcv(txs.ohlcv) ]); - // has to split this out because they are sharing the same table, will clash when inserting - await this.insertLiquidityOps(txs.withdrawLiquidityOpsData); - } - - private async queryLpOps(): Promise { - return this.duckDb.queryLpOps() as Promise; } private async getPoolInfos(pairAddrs: string[], wantedHeight?: number): Promise { @@ -80,22 +55,6 @@ class WriteOrders extends WriteData { collectAccumulateLpData(data, poolInfos); } - // private insertVolumeInfos( - // ...data: { denom: string; timestamp: number; txheight: number; amount: number }[] - // ): VolumeInfo[] { - // let volumeInfos: VolumeInfo[] = []; - // data.forEach((op) => { - // volumeInfos.push({ - // denom: op.denom, - // timestamp: op.timestamp, - // txheight: op.txheight, - // volume: op.amount, - // price: 1 - // }); - // }); - // return volumeInfos; - // } - async process(chunk: any): Promise { try { // // first time calling of the application then we query past data and be ready to store them into the db for prefix sum @@ -133,7 +92,7 @@ class WriteOrders extends WriteData { // hash to be promise all because if inserting height pass and txs fail then we will have duplications await Promise.all([this.duckDb.insertHeightSnapshot(newOffset), this.insertParsedTxs(result)]); - const lpOps = await this.queryLpOps(); + const lpOps = await this.duckDb.queryLpOps(); const swapOpsCount = await this.duckDb.querySwapOps(); console.log("lp ops: ", lpOps.length); console.log("swap ops: ", swapOpsCount); @@ -170,18 +129,6 @@ class OraiDexSync { return getAllPairInfos(firstFactoryClient, secondFactoryClient); } - private async simulateSwapPrice(info: AssetInfo, wantedHeight?: number): Promise { - // adjust the query height to get data from the past - this.cosmwasmClient.setQueryClientWithHeight(wantedHeight); - const routerContract = new OraiswapRouterQueryClient( - this.cosmwasmClient, - this.env.ROUTER_CONTRACT_ADDRESS || "orai1j0r67r9k8t34pnhy00x3ftuxuwg0r6r4p8p6rrc8az0ednzr8y9s3sj2sf" - ); - const data = await simulateSwapPriceWithUsdt(info, routerContract); - this.cosmwasmClient.setQueryClientWithHeight(); - return data; - } - private async updateLatestPairInfos() { const pairInfos = await this.getAllPairInfos(); await this.duckDb.insertPairInfos( @@ -206,8 +153,8 @@ class OraiDexSync { this.duckDb.createLiquidityOpsTable(), this.duckDb.createSwapOpsTable(), this.duckDb.createPairInfosTable(), - this.duckDb.createPriceInfoTable(), - this.duckDb.createVolumeInfo() + // this.duckDb.createPriceInfoTable(), + this.duckDb.createSwapOhlcv() ]); let currentInd = await this.duckDb.loadHeightSnapshot(); let initialData: InitialData = { tokenPrices: [], blockHeader: undefined }; diff --git a/packages/oraidex-sync/src/pairs.ts b/packages/oraidex-sync/src/pairs.ts index 49ee967c..40c47553 100644 --- a/packages/oraidex-sync/src/pairs.ts +++ b/packages/oraidex-sync/src/pairs.ts @@ -16,6 +16,7 @@ import { usdtCw20Address } from "./constants"; import { PairMapping } from "./types"; +import { parseAssetInfoOnlyDenom } from "./helper"; // the orders are important! Do not change the order of the asset_infos. export const pairs: PairMapping[] = [ @@ -88,4 +89,9 @@ export function extractUniqueAndFlatten(data: PairMapping[]): AssetInfo[] { return uniqueFlattenedArray; } +export const pairsOnlyDenom = pairs.map((pair) => ({ + ...pair, + asset_infos: pair.asset_infos.map((info) => parseAssetInfoOnlyDenom(info)) +})); + export const uniqueInfos = extractUniqueAndFlatten(pairs); diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index 740c0903..eb5adc68 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -11,17 +11,16 @@ import { MsgType, OraiswapPairCw20HookMsg, OraiswapRouterCw20HookMsg, - PrefixSumHandlingData, ProvideLiquidityOperationData, SwapOperationData, TxAnlysisResult, - VolumeInfo, WithdrawLiquidityOperationData } from "./types"; import { Log } from "@cosmjs/stargate/build/logs"; import { - calculatePrefixSum, + buildOhlcv, concatDataToUniqueKey, + getSwapDirection, groupByTime, isAssetInfoPairReverse, isoToTimestampNumber, @@ -100,6 +99,7 @@ function extractSwapOperations(txData: BasicTxData, wasmAttributes: (readonly At swapData.push({ askDenom: askDenoms[i], commissionAmount: parseInt(commissionAmounts[i]), + direction: getSwapDirection(offerDenoms[i], askDenoms[i]), offerAmount, offerDenom: offerDenoms[i], uniqueKey: concatDataToUniqueKey({ @@ -277,10 +277,11 @@ function parseTxs(txs: Tx[]): TxAnlysisResult { accountTxs.push({ txhash: basicTxData.txhash, accountAddress: sender }); } } + swapOpsData = removeOpsDuplication(swapOpsData) as SwapOperationData[]; return { // transactions: txs, - swapOpsData: groupByTime(removeOpsDuplication(swapOpsData)) as SwapOperationData[], - volumeInfos: [], + swapOpsData: groupByTime(swapOpsData) as SwapOperationData[], + ohlcv: buildOhlcv(swapOpsData), accountTxs, provideLiquidityOpsData: groupByTime( removeOpsDuplication(provideLiquidityOpsData) diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 06171cdc..7eeb854c 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -4,6 +4,8 @@ import { Addr, Asset, AssetInfo, Binary, Decimal, SwapOperation, Uint128 } from import { ExecuteMsg as OraiswapRouterExecuteMsg } from "@oraichain/oraidex-contracts-sdk/build/OraiswapRouter.types"; import { MsgExecuteContract } from "cosmjs-types/cosmwasm/wasm/v1/tx"; +export type SwapDirection = "Buy" | "Sell"; + export type BasicTxData = { timestamp: number; txhash: string; @@ -13,6 +15,7 @@ export type BasicTxData = { export type SwapOperationData = { askDenom: string; // eg: orai, orai1234... commissionAmount: number; + direction: SwapDirection; offerAmount: number; offerDenom: string; uniqueKey: string; // concat of offer, ask denom, amount, and timestamp => should be unique @@ -73,7 +76,7 @@ export type OraiDexType = SwapOperationData | ProvideLiquidityOperationData | Wi export type TxAnlysisResult = { // transactions: Tx[]; swapOpsData: SwapOperationData[]; - volumeInfos: VolumeInfo[]; + ohlcv: Ohlcv[]; accountTxs: AccountTx[]; provideLiquidityOpsData: ProvideLiquidityOperationData[]; withdrawLiquidityOpsData: WithdrawLiquidityOperationData[]; @@ -127,11 +130,6 @@ export type InitialData = { blockHeader: BlockHeader; }; -export type PrefixSumHandlingData = { - denom: string; - amount: number; -}; - export type TickerInfo = { base_currency: string; target_currency: string; @@ -150,14 +148,6 @@ export type TotalLiquidity = { height: number; }; -export type VolumeInfo = { - denom: string; - timestamp: number; - txheight: number; - price: number; - volume: number; -}; - export type Env = { PORT: number; RPC_URL: string; @@ -171,3 +161,17 @@ export type Env = { DUCKDB_FILENAME: string; INITIAL_SYNC_HEIGHT: number; }; + +export interface TradeItem { + timestamp: number; + pair: string; + price?: number; + volume: bigint; +} + +export interface Ohlcv extends TradeItem { + open: number; + close: number; + low: number; + high: number; +} diff --git a/packages/oraidex-sync/tests/db.spec.ts b/packages/oraidex-sync/tests/db.spec.ts index ec6be119..95741564 100644 --- a/packages/oraidex-sync/tests/db.spec.ts +++ b/packages/oraidex-sync/tests/db.spec.ts @@ -23,6 +23,7 @@ describe("test-duckdb", () => { { askDenom: "orai", commissionAmount: 0, + direction: "Buy", offerAmount: 10000, offerDenom: "atom", uniqueKey: "1", @@ -36,6 +37,7 @@ describe("test-duckdb", () => { { askDenom: "orai", commissionAmount: 0, + direction: "Buy", offerAmount: 10, offerDenom: "atom", uniqueKey: "2", @@ -49,6 +51,7 @@ describe("test-duckdb", () => { { askDenom: "atom", commissionAmount: 0, + direction: "Sell", offerAmount: 10, offerDenom: "orai", uniqueKey: "3", @@ -62,6 +65,7 @@ describe("test-duckdb", () => { { askDenom: "atom", commissionAmount: 0, + direction: "Sell", offerAmount: 10, offerDenom: "orai", uniqueKey: "4", @@ -86,6 +90,7 @@ describe("test-duckdb", () => { { askDenom: "orai", commissionAmount: 0, + direction: "Buy", offerAmount: 10000, offerDenom: "atom", uniqueKey: "1", @@ -99,6 +104,7 @@ describe("test-duckdb", () => { { askDenom: "atom", commissionAmount: 0, + direction: "Sell", offerAmount: 10, offerDenom: "orai", uniqueKey: "2", @@ -112,6 +118,7 @@ describe("test-duckdb", () => { { askDenom: "orai", commissionAmount: 0, + direction: "Buy", offerAmount: 100000, offerDenom: "atom", uniqueKey: "3", @@ -125,6 +132,7 @@ describe("test-duckdb", () => { { askDenom: "atom", commissionAmount: 0, + direction: "Sell", offerAmount: 1000000, offerDenom: "orai", uniqueKey: "4", diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index f8c0cc28..fdc39eac 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -1,6 +1,5 @@ import { AssetInfo } from "@oraichain/oraidex-contracts-sdk"; import { - calculatePrefixSum, findAssetInfoPathToUsdt, findMappedTargetedAssetInfo, findPairAddress, @@ -12,7 +11,11 @@ import { groupByTime, collectAccumulateLpData, concatDataToUniqueKey, - removeOpsDuplication + removeOpsDuplication, + calculateOhlcv, + calculatePriceFromSwapOp, + getSwapDirection, + findPairIndexFromDenoms } from "../src/helper"; import { extractUniqueAndFlatten, pairs } from "../src/pairs"; import { @@ -29,7 +32,7 @@ import { usdcCw20Address, usdtCw20Address } from "../src/constants"; -import { PairInfoData, ProvideLiquidityOperationData } from "../src/types"; +import { PairInfoData, ProvideLiquidityOperationData, SwapDirection, SwapOperationData } from "../src/types"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; describe("test-helper", () => { @@ -142,27 +145,6 @@ describe("test-helper", () => { expect(result.length).toEqual(expectedListLength); }); - it("test-calculatePrefixSum", () => { - const data = [ - { - denom: "foo", - amount: 100 - }, - { denom: "foo", amount: 10 }, - { denom: "bar", amount: 5 }, - { denom: "bar", amount: -1 }, - { denom: "hello", amount: 5 } - ]; - const result = calculatePrefixSum(1, data); - expect(result).toEqual([ - { denom: "foo", amount: 101 }, - { denom: "foo", amount: 111 }, - { denom: "bar", amount: 6 }, - { denom: "bar", amount: 5 }, - { denom: "hello", amount: 6 } - ]); - }); - it("test-extractUniqueAndFlatten-extracting-unique-items-in-pair-mapping", () => { // act const result = extractUniqueAndFlatten(pairs); @@ -532,4 +514,85 @@ describe("test-helper", () => { // expect(result.target).toEqual(expectedInfo); // expect(result.baseIndex).toEqual(expectedBase); // }); + + it.each([ + [ + [ + { + timestamp: 60000, + pair: "orai-usdt", + price: 1, + volume: BigInt(100) + }, + { + timestamp: 60000, + pair: "orai-usdt", + price: 2, + volume: BigInt(100) + } + ], + { + open: 1, + close: 2, + low: 1, + high: 2, + volume: BigInt(200), + timestamp: 1, + pair: "orai-usdt" + } + ] + ])("test-calculateOhlcv", (orders, expectedOhlcv) => { + const ohlcv = calculateOhlcv(orders); + expect(ohlcv).toEqual(expectedOhlcv); + }); + + it.each([ + ["Buy" as SwapDirection, 2], + ["Sell" as SwapDirection, 0.5] + ])("test-calculatePriceFromOrder", (direction: SwapDirection, expectedPrice: number) => { + const swapOp = { + offerAmount: 1, + offerDenom: ORAI, + returnAmount: 1, + askDenom: usdtCw20Address, + direction, + uniqueKey: "1", + timestamp: 1, + txCreator: "a", + txhash: "a", + txheight: 1, + spreadAmount: 1, + taxAmount: 1, + commissionAmount: 1 + } as SwapOperationData; + // first case undefined, return 0 + expect(calculatePriceFromSwapOp(undefined as any)).toEqual(0); + // other cases + const price = calculatePriceFromSwapOp(swapOp); + expect(price).toEqual(expectedPrice); + }); + + it.each([ + [usdtCw20Address, "orai", "Buy" as SwapDirection], + ["orai", usdtCw20Address, "Sell" as SwapDirection] + ])("test-getSwapDirection", (offerDenom: string, askDenom: string, expectedDirection: SwapDirection) => { + // execute + // throw error case when offer & ask not in pair + expect(getSwapDirection("foo", "bar")).toThrow(); + const result = getSwapDirection(offerDenom, askDenom); + expect(result).toEqual(expectedDirection); + }); + + it.each([ + ["orai", usdtCw20Address, 4], + [usdtCw20Address, "orai", 4], + ["orai", airiCw20Adress, 0], + ["orai", "foo", -1] + ])( + "test-findPairIndexFromDenoms-given-%s-and-%s-should-return-index-%d-from-pair-list", + (offerDenom: string, askDenom: string, expectedIndex: number) => { + const result = findPairIndexFromDenoms(offerDenom, askDenom); + expect(result).toEqual(expectedIndex); + } + ); }); diff --git a/yarn.lock b/yarn.lock index eae19908..03de09aa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1952,6 +1952,11 @@ dependencies: "@cosmjs/stargate" "^0.31.0" +"@oraichain/oraidex-contracts-sdk@^1.0.18": + version "1.0.18" + resolved "https://registry.yarnpkg.com/@oraichain/oraidex-contracts-sdk/-/oraidex-contracts-sdk-1.0.18.tgz#6dd983f4b13b6cbb83060ae3b0abc90ec4133624" + integrity sha512-f8huSEVdormJXzltBgD2jjvBmL6PfOzP+sz9IpZ2yV8zB4dzjRMgnT+39Vfp5CYARKQlbe5Hdk/0cD2kQLxKjA== + "@parcel/watcher@2.0.4": version "2.0.4" resolved "https://registry.yarnpkg.com/@parcel/watcher/-/watcher-2.0.4.tgz#f300fef4cc38008ff4b8c29d92588eced3ce014b" From 1d46f0de214fe9169ae2ac563debd49bb8791438 Mon Sep 17 00:00:00 2001 From: ducphamle2 Date: Fri, 28 Jul 2023 10:14:49 +0700 Subject: [PATCH 3/5] finished swap ohlcv --- packages/oraidex-sync/src/db.ts | 42 +++++++----- packages/oraidex-sync/src/helper.ts | 67 +++++++++++-------- packages/oraidex-sync/src/index.ts | 3 +- packages/oraidex-sync/src/pairs.ts | 6 +- packages/oraidex-sync/src/test-db.ts | 14 ++-- packages/oraidex-sync/src/tx-parsing.ts | 6 +- packages/oraidex-sync/src/types.ts | 15 ++--- packages/oraidex-sync/tests/helper.spec.ts | 77 ++++++++++++---------- 8 files changed, 127 insertions(+), 103 deletions(-) diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index 90561496..011ca23d 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -288,9 +288,9 @@ export class DuckDb { async createSwapOhlcv() { await this.conn.exec( `CREATE TABLE IF NOT EXISTS swap_ohlcv ( + uniqueKey varchar UNIQUE, timestamp uinteger, pair varchar, - price double, volume ubigint, open double, close double, @@ -304,23 +304,31 @@ export class DuckDb { await this.insertBulkData(ohlcv, "swap_ohlcv"); } - async pivotVolumeRange(startTime: number, endTime: number) { - let volumeInfos = await this.conn.all( + async getOhlcvCandles(pair: string, tf: number, startTime: number, endTime: number): Promise { + // tf should be in seconds + const result = await this.conn.all( ` - pivot (select * from swap_ohlcv - where timestamp >= ${startTime} - and timestamp <= ${endTime} - order by timestamp) - on denom - using sum(volume) - group by timestamp, txheight - order by timestamp` + SELECT timestamp // ? as time, + sum(volume) as volume, + first(open) as open, + last(close) as close, + min(low) as low, + max(high) as high + FROM swap_ohlcv + WHERE pair = ? AND timestamp >= ? AND timestamp <= ? + GROUP BY time + ORDER BY time + `, + +tf, + pair, + startTime, + endTime ); - for (let volInfo of volumeInfos) { - for (const key in volInfo) { - if (volInfo[key] === null) volInfo[key] = 0; - } - } - return volumeInfos; + + // get second + result.forEach((item) => { + item.time *= Number(tf); + }); + return result as Ohlcv[]; } } diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index eeb1e510..e6e27a5e 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -8,13 +8,12 @@ import { ProvideLiquidityOperationData, SwapDirection, SwapOperationData, - TradeItem, WithdrawLiquidityOperationData } from "./types"; import { PoolResponse } from "@oraichain/oraidex-contracts-sdk/build/OraiswapPair.types"; import { minBy, maxBy } from "lodash"; -export function toObject(data: any[]) { +export function toObject(data: any) { return JSON.parse( JSON.stringify( data, @@ -70,9 +69,13 @@ export function concatDataToUniqueKey(data: { secondDenom: string; firstAmount: number; secondAmount: number; - timestamp: number; + txheight: number; }): string { - return `${data.timestamp}-${data.firstDenom}-${data.firstAmount}-${data.secondDenom}-${data.secondAmount}`; + return `${data.txheight}-${data.firstDenom}-${data.firstAmount}-${data.secondDenom}-${data.secondAmount}`; +} + +export function concatOhlcvToUniqueKey(data: { timestamp: number; pair: string; volume: bigint }): string { + return `${data.timestamp}-${data.pair}-${data.volume.toString()}`; } export function isoToTimestampNumber(time: string) { @@ -161,7 +164,7 @@ function calculatePriceByPool(offerPool: bigint, askPool: bigint, commissionRate return (askPool - (offerPool * askPool) / (offerPool + BigInt(tenAmountInDecimalSix))) * BigInt(1 - commissionRate); } -export function groupByTime(data: any[], timeframe?: number): any[] { +export function groupDataByTime(data: any[], timeframe?: number): { [key: string]: any[] } { let ops: { [k: number]: any[] } = {}; for (const op of data) { const roundedTime = roundTime(op.timestamp * 1000, timeframe || 60); // op timestamp is sec @@ -175,7 +178,11 @@ export function groupByTime(data: any[], timeframe?: number): any[] { ops[roundedTime].push(newData); } - return Object.values(ops).flat(); + return ops; +} + +export function groupByTime(data: any[], timeframe?: number): any[] { + return Object.values(groupDataByTime(data, timeframe)).flat(); } /** @@ -259,7 +266,8 @@ export function groupSwapOpsByPair(ops: SwapOperationData[]): { [key: string]: S for (const op of ops) { const pairIndex = findPairIndexFromDenoms(op.offerDenom, op.askDenom); if (pairIndex === -1) continue; - const pair = JSON.stringify(pairs[pairIndex].asset_infos); + const assetInfos = pairsOnlyDenom[pairIndex].asset_infos; + const pair = `${assetInfos[0]}-${assetInfos[1]}`; if (!opsByPair[pair]) { opsByPair[pair] = []; } @@ -268,33 +276,36 @@ export function groupSwapOpsByPair(ops: SwapOperationData[]): { [key: string]: S return opsByPair; } -export function calculateOhlcv(orders: TradeItem[]): Ohlcv { - const timestamp = orders[0].timestamp; - const pair = orders[0].pair; - const open = orders[0].price; - const close = orders[orders.length - 1].price; - const low = minBy(orders, "price").price; - const high = maxBy(orders, "price").price; - const volume = orders.reduce((acc, currentValue) => { - return acc + currentValue.volume; - }, BigInt(0)); +export function calculateSwapOhlcv(ops: SwapOperationData[], pair: string): Ohlcv { + const timestamp = ops[0].timestamp; + const prices = ops.map((op) => calculatePriceFromSwapOp(op)); + const open = prices[0]; + const close = prices[ops.length - 1]; + const low = minBy(prices); + const high = maxBy(prices); + const volume = ops.reduce((acc, currentValue) => { + return acc + currentValue.direction === "Buy" + ? BigInt(currentValue.offerAmount) + : BigInt(currentValue.returnAmount); + }, 0n); return { + uniqueKey: concatOhlcvToUniqueKey({ timestamp, pair, volume }), + timestamp, + pair, + volume, open, close, low, - high, - volume, - timestamp, - pair + high }; } export function buildOhlcv(ops: SwapOperationData[]): Ohlcv[] { let ohlcv: Ohlcv[] = []; - for (const [_, opsByPair] of Object.entries(groupSwapOpsByPair(ops))) { - const orderByTimes = groupByTime(opsByPair); - const ticks = Object.values(orderByTimes).map((value) => calculateOhlcv(value)); + for (const [pair, opsByPair] of Object.entries(groupSwapOpsByPair(ops))) { + const opsByTime = groupDataByTime(opsByPair); + const ticks = Object.values(opsByTime).map((value) => calculateSwapOhlcv(value, pair)); ohlcv.push(...ticks); } return ohlcv; @@ -310,11 +321,11 @@ export function calculatePriceFromSwapOp(op: SwapOperationData): number { } export function getSwapDirection(offerDenom: string, askDenom: string): SwapDirection { - const pair = pairsOnlyDenom.find( - (pair) => pair.asset_infos.some((info) => info === offerDenom) && pair.asset_infos.some((info) => info === askDenom) - ); + const pair = pairsOnlyDenom.find((pair) => { + return pair.asset_infos.some((info) => info === offerDenom) && pair.asset_infos.some((info) => info === askDenom); + }); if (!pair) { - throw Error("Cannot find asset infos in list of pairs"); + throw new Error("Cannot find asset infos in list of pairs"); } const assetInfos = pair.asset_infos; // use quote denom as offer then its buy. Quote denom in pairs is the 2nd index in the array diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index 5c059e11..d7e0f4ce 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -27,9 +27,10 @@ class WriteOrders extends WriteData { // insert swap ops await Promise.all([ this.duckDb.insertSwapOps(txs.swapOpsData), - this.duckDb.insertLpOps([...txs.provideLiquidityOpsData, ...txs.withdrawLiquidityOpsData]), + this.duckDb.insertLpOps(txs.provideLiquidityOpsData), this.duckDb.insertOhlcv(txs.ohlcv) ]); + await this.duckDb.insertLpOps(txs.withdrawLiquidityOpsData); } private async getPoolInfos(pairAddrs: string[], wantedHeight?: number): Promise { diff --git a/packages/oraidex-sync/src/pairs.ts b/packages/oraidex-sync/src/pairs.ts index 40c47553..33758dd4 100644 --- a/packages/oraidex-sync/src/pairs.ts +++ b/packages/oraidex-sync/src/pairs.ts @@ -16,7 +16,6 @@ import { usdtCw20Address } from "./constants"; import { PairMapping } from "./types"; -import { parseAssetInfoOnlyDenom } from "./helper"; // the orders are important! Do not change the order of the asset_infos. export const pairs: PairMapping[] = [ @@ -91,7 +90,10 @@ export function extractUniqueAndFlatten(data: PairMapping[]): AssetInfo[] { export const pairsOnlyDenom = pairs.map((pair) => ({ ...pair, - asset_infos: pair.asset_infos.map((info) => parseAssetInfoOnlyDenom(info)) + asset_infos: pair.asset_infos.map((info) => { + if ("native_token" in info) return info.native_token.denom; + return info.token.contract_addr; + }) })); export const uniqueInfos = extractUniqueAndFlatten(pairs); diff --git a/packages/oraidex-sync/src/test-db.ts b/packages/oraidex-sync/src/test-db.ts index 0a0ee39f..1db28ef4 100644 --- a/packages/oraidex-sync/src/test-db.ts +++ b/packages/oraidex-sync/src/test-db.ts @@ -2,18 +2,18 @@ import { CosmWasmClient, OraiswapRouterQueryClient, SwapOperation } from "@oraic import { DuckDb } from "./db"; import { SwapOperationData } from "./types"; import { pairs, uniqueInfos } from "./pairs"; -import { groupByTime, parseAssetInfoOnlyDenom } from "./helper"; +import { parseAssetInfoOnlyDenom } from "./helper"; import { simulateSwapPriceWithUsdt } from "./query"; import "dotenv/config"; const start = async () => { - const duckdb = await DuckDb.create("oraidex-sync-data-v1.2"); + const duckdb = await DuckDb.create("oraidex-sync-data-staging"); const tf = 86400; - const firstTokenResult = await duckdb.conn.all( - `SELECT * - from swap_ops_data - where timestamp >= 1690168508 and timestamp <= 1690169408 and askDenom = 'ibc/A2E2EEC9057A4A1C2C0A6A4C78B0239118DF5F278830F50B4A6BDD7A66506B78' - order by timestamp` + const firstTokenResult = await duckdb.getOhlcvCandles( + "orai-orai12hzjxfh77wl572gdzct2fxv2arxcwh6gykc7qh", + 60, + 1688789160, + 1688796360 ); console.log(firstTokenResult); diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index eb5adc68..3614257e 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -103,7 +103,7 @@ function extractSwapOperations(txData: BasicTxData, wasmAttributes: (readonly At offerAmount, offerDenom: offerDenoms[i], uniqueKey: concatDataToUniqueKey({ - timestamp: txData.timestamp, + txheight: txData.txheight, firstAmount: offerAmount, firstDenom: offerDenoms[i], secondAmount: returnAmount, @@ -145,7 +145,7 @@ function extractMsgProvideLiquidity( firstTokenLp: firstAmount, opType: "provide", uniqueKey: concatDataToUniqueKey({ - timestamp: txData.timestamp, + txheight: txData.txheight, firstAmount, firstDenom, secondAmount: secAmount, @@ -203,7 +203,7 @@ function extractMsgWithdrawLiquidity( firstTokenLp: parseInt(assets[0]), opType: "withdraw", uniqueKey: concatDataToUniqueKey({ - timestamp: txData.timestamp, + txheight: txData.txheight, firstDenom: assets[1], firstAmount: parseInt(assets[0]), secondDenom: assets[3], diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 7eeb854c..8d4143f3 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -16,10 +16,10 @@ export type SwapOperationData = { askDenom: string; // eg: orai, orai1234... commissionAmount: number; direction: SwapDirection; - offerAmount: number; + offerAmount: number | bigint; offerDenom: string; uniqueKey: string; // concat of offer, ask denom, amount, and timestamp => should be unique - returnAmount: number; + returnAmount: number | bigint; spreadAmount: number; taxAmount: number; } & BasicTxData; @@ -65,13 +65,13 @@ export type ProvideLiquidityOperationData = { secondTokenDenom: string; secondTokenLp: number | bigint; opType: LiquidityOpType; - uniqueKey: string; // concat of first, second denom, amount, and timestamp => should be unique + uniqueKey: string; // concat of first, second denom, amount, and timestamp => should be unique. unique key is used to override duplication only. txCreator: string; } & BasicTxData; export type WithdrawLiquidityOperationData = ProvideLiquidityOperationData; -export type OraiDexType = SwapOperationData | ProvideLiquidityOperationData | WithdrawLiquidityOperationData; +export type OraiDexType = SwapOperationData | ProvideLiquidityOperationData | WithdrawLiquidityOperationData | Ohlcv; export type TxAnlysisResult = { // transactions: Tx[]; @@ -162,14 +162,11 @@ export type Env = { INITIAL_SYNC_HEIGHT: number; }; -export interface TradeItem { +export interface Ohlcv { + uniqueKey: string; // concat of timestamp, pair and volume. Only use to override potential duplication when inserting timestamp: number; pair: string; - price?: number; volume: bigint; -} - -export interface Ohlcv extends TradeItem { open: number; close: number; low: number; diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index fdc39eac..9216713d 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -12,10 +12,11 @@ import { collectAccumulateLpData, concatDataToUniqueKey, removeOpsDuplication, - calculateOhlcv, calculatePriceFromSwapOp, getSwapDirection, - findPairIndexFromDenoms + findPairIndexFromDenoms, + toObject, + calculateSwapOhlcv } from "../src/helper"; import { extractUniqueAndFlatten, pairs } from "../src/pairs"; import { @@ -427,10 +428,10 @@ describe("test-helper", () => { const firstAmount = 1; const secondDenom = "bar"; const secondAmount = 1; - const timestamp = 100; + const txheight = 100; // act - const result = concatDataToUniqueKey({ firstAmount, firstDenom, secondAmount, secondDenom, timestamp }); + const result = concatDataToUniqueKey({ firstAmount, firstDenom, secondAmount, secondDenom, txheight }); // assert expect(result).toEqual("100-foo-1-bar-1"); @@ -515,43 +516,43 @@ describe("test-helper", () => { // expect(result.baseIndex).toEqual(expectedBase); // }); - it.each([ - [ - [ - { - timestamp: 60000, - pair: "orai-usdt", - price: 1, - volume: BigInt(100) - }, - { - timestamp: 60000, - pair: "orai-usdt", - price: 2, - volume: BigInt(100) - } - ], - { - open: 1, - close: 2, - low: 1, - high: 2, - volume: BigInt(200), - timestamp: 1, - pair: "orai-usdt" - } - ] - ])("test-calculateOhlcv", (orders, expectedOhlcv) => { - const ohlcv = calculateOhlcv(orders); - expect(ohlcv).toEqual(expectedOhlcv); - }); + // it.each([ + // [ + // [ + // { + // timestamp: 60000, + // pair: "orai-usdt", + // price: 1, + // volume: 100n + // }, + // { + // timestamp: 60000, + // pair: "orai-usdt", + // price: 2, + // volume: 100n + // } + // ], + // { + // open: 1, + // close: 2, + // low: 1, + // high: 2, + // volume: 200n, + // timestamp: 60000, + // pair: "orai-usdt" + // } + // ] + // ])("test-calculateOhlcv", (ops, expectedOhlcv) => { + // const ohlcv = calculateSwapOhlcv(ops); + // expect(toObject(ohlcv)).toEqual(toObject(expectedOhlcv)); + // }); it.each([ ["Buy" as SwapDirection, 2], ["Sell" as SwapDirection, 0.5] ])("test-calculatePriceFromOrder", (direction: SwapDirection, expectedPrice: number) => { const swapOp = { - offerAmount: 1, + offerAmount: 2, offerDenom: ORAI, returnAmount: 1, askDenom: usdtCw20Address, @@ -578,7 +579,11 @@ describe("test-helper", () => { ])("test-getSwapDirection", (offerDenom: string, askDenom: string, expectedDirection: SwapDirection) => { // execute // throw error case when offer & ask not in pair - expect(getSwapDirection("foo", "bar")).toThrow(); + try { + getSwapDirection("foo", "bar"); + } catch (error) { + expect(error).toEqual(new Error("Cannot find asset infos in list of pairs")); + } const result = getSwapDirection(offerDenom, askDenom); expect(result).toEqual(expectedDirection); }); From 64ff0df24a92431ec06bd0bdd1f9fd90db54978c Mon Sep 17 00:00:00 2001 From: ducphamle2 Date: Sat, 29 Jul 2023 22:47:58 +0700 Subject: [PATCH 4/5] renamed lp ops cols and updated query total volume api --- packages/oraidex-server/src/helper.ts | 20 ++++ packages/oraidex-server/src/index.ts | 103 +++++++++------------ packages/oraidex-sync/src/constants.ts | 2 +- packages/oraidex-sync/src/db.ts | 44 +++++++-- packages/oraidex-sync/src/helper.ts | 66 ++++++++----- packages/oraidex-sync/src/pairs.ts | 4 + packages/oraidex-sync/src/query.ts | 2 +- packages/oraidex-sync/src/test-db.ts | 17 ++-- packages/oraidex-sync/src/tx-parsing.ts | 64 +++++++------ packages/oraidex-sync/src/types.ts | 25 +++-- packages/oraidex-sync/tests/db.spec.ts | 39 ++++---- packages/oraidex-sync/tests/helper.spec.ts | 97 ++++++++++--------- 12 files changed, 283 insertions(+), 200 deletions(-) diff --git a/packages/oraidex-server/src/helper.ts b/packages/oraidex-server/src/helper.ts index 44784e58..b25ad875 100644 --- a/packages/oraidex-server/src/helper.ts +++ b/packages/oraidex-server/src/helper.ts @@ -7,3 +7,23 @@ export function getDate24hBeforeNow(time: Date) { const date24hBeforeNow = new Date(time.getTime() - twentyFourHoursInMilliseconds); return date24hBeforeNow; } + +/** + * + * @param time + * @param tf in seconds + * @returns + */ +export function getSpecificDateBeforeNow(time: Date, tf: number) { + const timeInMs = tf * 1000; // 24 hours in milliseconds + const dateBeforeNow = new Date(time.getTime() - timeInMs); + return dateBeforeNow; +} + +export function calculateBasePriceFromTickerVolume(baseVolume: string, targetVolume: string): number { + return parseFloat(targetVolume) / parseFloat(baseVolume); +} + +export function pairToString(pair: string[]): string { + return `${pair[0]}-${pair[1]}`; +} diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index 3cb6d2ee..0a824297 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -9,16 +9,15 @@ import { toDisplay, OraiDexSync, simulateSwapPrice, - getPoolInfos, - calculatePrefixSum, - uniqueInfos, - simulateSwapPriceWithUsdt + pairsOnlyDenom, + VolumeRange, + oraiUsdtPairOnlyDenom, + ORAI } from "@oraichain/oraidex-sync"; import cors from "cors"; import { CosmWasmClient } from "@cosmjs/cosmwasm-stargate"; -import { OraiswapRouterQueryClient, PairInfo } from "@oraichain/oraidex-contracts-sdk"; -import { getDate24hBeforeNow, parseSymbolsToTickerId } from "./helper"; -import { MulticallQueryClient } from "@oraichain/common-contracts-sdk"; +import { OraiswapRouterQueryClient } from "@oraichain/oraidex-contracts-sdk"; +import { getDate24hBeforeNow, getSpecificDateBeforeNow, pairToString, parseSymbolsToTickerId } from "./helper"; dotenv.config(); @@ -59,6 +58,8 @@ app.get("/tickers", async (req, res) => { process.env.ROUTER_CONTRACT_ADDRESS || "orai1j0r67r9k8t34pnhy00x3ftuxuwg0r6r4p8p6rrc8az0ednzr8y9s3sj2sf" ); const pairInfos = await duckDb.queryPairInfos(); + const latestTimestamp = endTime ? parseInt(endTime as string) : await duckDb.queryLatestTimestampSwapOps(); + const then = getDate24hBeforeNow(new Date(latestTimestamp * 1000)).getTime() / 1000; const data: TickerInfo[] = ( await Promise.allSettled( pairs.map(async (pair) => { @@ -68,8 +69,6 @@ app.get("/tickers", async (req, res) => { // const { baseIndex, targetIndex, target } = findUsdOraiInPair(pair.asset_infos); const baseIndex = 0; const targetIndex = 1; - const latestTimestamp = endTime ? parseInt(endTime as string) : await duckDb.queryLatestTimestampSwapOps(); - const then = getDate24hBeforeNow(new Date(latestTimestamp * 1000)).getTime() / 1000; console.log(latestTimestamp, then); const baseInfo = parseAssetInfoOnlyDenom(pair.asset_infos[baseIndex]); const targetInfo = parseAssetInfoOnlyDenom(pair.asset_infos[targetIndex]); @@ -110,59 +109,47 @@ app.get("/tickers", async (req, res) => { // TODO: refactor this and add unit tests app.get("/volume/v2/historical/chart", async (req, res) => { const { startTime, endTime, tf } = req.query; - const timeFrame = parseInt(tf as string); - const volumeInfos = await duckDb.pivotVolumeRange(parseInt(startTime as string), parseInt(endTime as string)); - const cosmwasmClient = await CosmWasmClient.connect(process.env.RPC_URL); - let finalArray = []; - let prices; - let heightCount = 0; - for (let i = 0; i < volumeInfos.length; i++) { - const volInfo = volumeInfos[i]; - cosmwasmClient.setQueryClientWithHeight(volInfo.txheight); - const router = new OraiswapRouterQueryClient( - cosmwasmClient, - process.env.ROUTER_CONTRACT_ADDRESS || "orai1j0r67r9k8t34pnhy00x3ftuxuwg0r6r4p8p6rrc8az0ednzr8y9s3sj2sf" - ); - if (heightCount % 1000 === 0) { - // prevent simulating too many times. TODO: calculate this using pool data from - prices = (await Promise.all(uniqueInfos.map((info) => simulateSwapPriceWithUsdt(info, router)))) - .map((price) => ({ ...price, info: parseAssetInfoOnlyDenom(price.info) })) - .reduce((acc, cur) => { - acc[cur.info] = parseFloat(cur.amount); - return acc; - }, {}); - } - let tempData = {}; - for (const key in volInfo) { - if (key === "timestamp" || key === "txheight") continue; - if (Object.keys(tempData).includes("volume_price")) { - tempData["volume_price"] += volInfo[key] * prices[key]; - } else { - tempData["timestamp"] = volInfo["timestamp"]; - tempData["volume_price"] = 0; - } - } - const indexOf = finalArray.findIndex((data) => data.timestamp === tempData["timestamp"]); - if (indexOf === -1) finalArray.push(tempData); - else { - finalArray[indexOf] = { - ...finalArray[indexOf], - volume_price: finalArray[indexOf].volume_price + tempData["volume_price"] - }; + const timeFrame = tf ? parseInt(tf as string) : 60; + const latestTimestamp = endTime ? parseInt(endTime as string) : await duckDb.queryLatestTimestampSwapOps(); + const then = startTime + ? parseInt(startTime as string) + : getSpecificDateBeforeNow(new Date(latestTimestamp * 1000), 259200).getTime() / 1000; + const volumeInfos = await Promise.all( + pairsOnlyDenom.map((pair) => { + return duckDb.getVolumeRange(timeFrame, then, latestTimestamp, pairToString(pair.asset_infos)); + }) + ); + // console.log("volume infos: ", volumeInfos); + let volumeRanges: { [time: string]: VolumeRange[] } = {}; + for (let volumePair of volumeInfos) { + for (let volume of volumePair) { + if (!volumeRanges[volume.time]) volumeRanges[volume.time] = [{ ...volume }]; + else volumeRanges[volume.time].push({ ...volume }); } - heightCount++; } - let finalFinalArray = []; - for (let data of finalArray) { - let time = Math.floor(data.timestamp / timeFrame); - let index = finalFinalArray.findIndex((data) => data.timestamp === time); - if (index === -1) { - finalFinalArray.push({ timestamp: time, volume_price: data.volume_price }); - } else { - finalFinalArray[index].volume_price += data.volume_price; + let result = []; + for (let [time, volumeData] of Object.entries(volumeRanges)) { + const oraiUsdtVolumeData = volumeData.find((data) => data.pair === pairToString(oraiUsdtPairOnlyDenom)); + if (!oraiUsdtVolumeData) { + res.status(500).send("Cannot find ORAI_USDT volume data in the volume list"); } + const totalVolumePrice = volumeData.reduce((acc, volData) => { + // console.log("base price in usdt: ", basePriceInUsdt); + // if base denom is orai then we calculate vol using quote vol + let volumePrice = 0; + if (volData.pair.split("-")[0] === ORAI) { + volumePrice = oraiUsdtVolumeData.basePrice * toDisplay(BigInt(volData.baseVolume)); + } else if (volData.pair.split("-")[1] === ORAI) { + volumePrice = oraiUsdtVolumeData.basePrice * toDisplay(BigInt(volData.quoteVolume)); + } else { + return acc; // skip for now cuz dont know how to calculate price if not paired if with ORAI + } + // volume price is calculated based on the base currency & quote volume + return acc + volumePrice; + }, 0); + result.push({ time, value: totalVolumePrice }); } - res.status(200).send(finalFinalArray); + res.status(200).send(result); }); // app.get("/liquidity/v2/historical/chart", async (req, res) => { diff --git a/packages/oraidex-sync/src/constants.ts b/packages/oraidex-sync/src/constants.ts index bc373754..69012d9d 100644 --- a/packages/oraidex-sync/src/constants.ts +++ b/packages/oraidex-sync/src/constants.ts @@ -10,6 +10,6 @@ export const scOraiCw20Address = "orai1065qe48g7aemju045aeyprflytemx7kecxkf5m7u5 export const usdcCw20Address = "orai15un8msx3n5zf9ahlxmfeqd2kwa5wm0nrpxer304m9nd5q6qq0g6sku5pdd"; export const atomIbcDenom = "ibc/A2E2EEC9057A4A1C2C0A6A4C78B0239118DF5F278830F50B4A6BDD7A66506B78"; export const osmosisIbcDenom = "ibc/9C4DCD21B48231D0BC2AC3D1B74A864746B37E4292694C93C617324250D002FC"; -export const tenAmountInDecimalSix = "10000000"; +export const tenAmountInDecimalSix = 10000000; export const truncDecimals = 6; export const atomic = 10 ** truncDecimals; diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index 011ca23d..d437fd23 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -7,6 +7,7 @@ import { TokenVolumeData, TotalLiquidity, VolumeData, + VolumeRange, WithdrawLiquidityOperationData } from "./types"; import fs, { rename } from "fs"; @@ -91,14 +92,15 @@ export class DuckDb { } await this.conn.exec( `CREATE TABLE IF NOT EXISTS lp_ops_data ( - firstTokenAmount UBIGINT, - firstTokenDenom VARCHAR, - firstTokenLp UBIGINT, + basePrice double, + baseTokenAmount UBIGINT, + baseTokenDenom VARCHAR, + baseTokenReserve UBIGINT, opType LPOPTYPE, uniqueKey VARCHAR UNIQUE, - secondTokenAmount UBIGINT, - secondTokenDenom VARCHAR, - secondTokenLp UBIGINT, + quoteTokenAmount UBIGINT, + quoteTokenDenom VARCHAR, + quoteTokenReserve UBIGINT, timestamp UINTEGER, txCreator VARCHAR, txhash VARCHAR, @@ -265,7 +267,7 @@ export class DuckDb { `with pivot_lp_ops as ( pivot lp_ops_data on opType - using sum(firstTokenAmount + secondTokenAmount) as liquidity ) + using sum(baseTokenAmount + quoteTokenAmount) as liquidity ) SELECT (timestamp // ?) as time, sum(COALESCE(provide_liquidity,0) - COALESCE(withdraw_liquidity, 0)) as liquidity, any_value(txheight) as height @@ -327,8 +329,34 @@ export class DuckDb { // get second result.forEach((item) => { - item.time *= Number(tf); + item.time *= tf; }); return result as Ohlcv[]; } + + async getVolumeRange(tf: number, startTime: number, endTime: number, pair: string): Promise { + const result = await this.conn.all( + ` + SELECT timestamp // ? as time, + any_value(pair) as pair, + sum(volume) as baseVolume, + cast(sum(close * volume) as UBIGINT) as quoteVolume, + avg(close) as basePrice, + FROM swap_ohlcv + WHERE timestamp >= ? + AND timestamp <= ? + and pair = ? + GROUP BY time + ORDER BY time + `, + tf, + startTime, + endTime, + pair + ); + return result.map((res) => ({ + ...res, + time: new Date(res.time * tf * 1000).toISOString() + })) as VolumeRange[]; + } } diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index e6e27a5e..adbe2cac 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -160,8 +160,17 @@ function findPairAddress(pairInfos: PairInfoData[], infos: [AssetInfo, AssetInfo )?.pairAddr; } -function calculatePriceByPool(offerPool: bigint, askPool: bigint, commissionRate: number): bigint { - return (askPool - (offerPool * askPool) / (offerPool + BigInt(tenAmountInDecimalSix))) * BigInt(1 - commissionRate); +function calculatePriceByPool( + basePool: bigint, + quotePool: bigint, + commissionRate?: number, + offerAmount?: number +): number { + const finalOfferAmount = offerAmount || tenAmountInDecimalSix; + let bigIntAmount = Number( + (basePool - (quotePool * basePool) / (quotePool + BigInt(finalOfferAmount))) * BigInt(1 - commissionRate || 0) + ); + return bigIntAmount / finalOfferAmount; } export function groupDataByTime(data: any[], timeframe?: number): { [key: string]: any[] } { @@ -208,43 +217,49 @@ export function isAssetInfoPairReverse(assetInfos: AssetInfo[]): boolean { * @param data - lp ops. This param will be mutated. * @param poolInfos - pool info data for initial lp accumulation */ +// TODO: write test cases for this function export function collectAccumulateLpData( data: ProvideLiquidityOperationData[] | WithdrawLiquidityOperationData[], poolInfos: PoolResponse[] ) { - let accumulateData = {}; + let accumulateData: { + [key: string]: { + baseTokenAmount: bigint; + quoteTokenAmount: bigint; + }; + } = {}; for (let op of data) { const pool = poolInfos.find( (info) => - info.assets.some((assetInfo) => parseAssetInfoOnlyDenom(assetInfo.info) === op.firstTokenDenom) && - info.assets.some((assetInfo) => parseAssetInfoOnlyDenom(assetInfo.info) === op.secondTokenDenom) + info.assets.some((assetInfo) => parseAssetInfoOnlyDenom(assetInfo.info) === op.baseTokenDenom) && + info.assets.some((assetInfo) => parseAssetInfoOnlyDenom(assetInfo.info) === op.quoteTokenDenom) ); if (!pool) continue; if (op.opType === "withdraw") { // reverse sign since withdraw means lp decreases - op.firstTokenLp = -BigInt(op.firstTokenLp); - op.secondTokenLp = -BigInt(op.secondTokenLp); + op.baseTokenReserve = -BigInt(op.baseTokenReserve); + op.quoteTokenReserve = -BigInt(op.quoteTokenReserve); } - const denom = `${op.firstTokenDenom} - ${op.secondTokenDenom}`; + const denom = `${op.baseTokenDenom}-${op.quoteTokenDenom}`; if (!accumulateData[denom]) { const initialFirstTokenAmount = parseInt( - pool.assets.find((info) => parseAssetInfoOnlyDenom(info.info) === op.firstTokenDenom).amount + pool.assets.find((info) => parseAssetInfoOnlyDenom(info.info) === op.baseTokenDenom).amount ); const initialSecondTokenAmount = parseInt( - pool.assets.find((info) => parseAssetInfoOnlyDenom(info.info) === op.secondTokenDenom).amount + pool.assets.find((info) => parseAssetInfoOnlyDenom(info.info) === op.quoteTokenDenom).amount ); accumulateData[denom] = { - firstTokenAmount: BigInt(initialFirstTokenAmount) + BigInt(op.firstTokenLp), - secondTokenAmount: BigInt(initialSecondTokenAmount) + BigInt(op.secondTokenLp) + baseTokenAmount: BigInt(initialFirstTokenAmount) + BigInt(op.baseTokenReserve), + quoteTokenAmount: BigInt(initialSecondTokenAmount) + BigInt(op.quoteTokenReserve) }; - op.firstTokenLp = accumulateData[denom].firstTokenAmount; - op.secondTokenLp = accumulateData[denom].secondTokenAmount; + op.baseTokenReserve = accumulateData[denom].baseTokenAmount; + op.quoteTokenReserve = accumulateData[denom].quoteTokenAmount; continue; } - accumulateData[denom].firstTokenAmount += BigInt(op.firstTokenLp); - accumulateData[denom].secondTokenAmount += BigInt(op.secondTokenLp); - op.firstTokenLp = accumulateData[denom].firstTokenAmount; - op.secondTokenLp = accumulateData[denom].secondTokenAmount; + accumulateData[denom].baseTokenAmount += BigInt(op.baseTokenReserve); + accumulateData[denom].quoteTokenAmount += BigInt(op.quoteTokenReserve); + op.baseTokenReserve = accumulateData[denom].baseTokenAmount; + op.quoteTokenReserve = accumulateData[denom].quoteTokenAmount; } } @@ -278,15 +293,16 @@ export function groupSwapOpsByPair(ops: SwapOperationData[]): { [key: string]: S export function calculateSwapOhlcv(ops: SwapOperationData[], pair: string): Ohlcv { const timestamp = ops[0].timestamp; - const prices = ops.map((op) => calculatePriceFromSwapOp(op)); + const prices = ops.map((op) => calculateBasePriceFromSwapOp(op)); const open = prices[0]; const close = prices[ops.length - 1]; const low = minBy(prices); const high = maxBy(prices); + // base volume const volume = ops.reduce((acc, currentValue) => { - return acc + currentValue.direction === "Buy" - ? BigInt(currentValue.offerAmount) - : BigInt(currentValue.returnAmount); + const baseVolume = + currentValue.direction === "Buy" ? BigInt(currentValue.returnAmount) : BigInt(currentValue.offerAmount); + return acc + baseVolume; }, 0n); return { @@ -311,7 +327,7 @@ export function buildOhlcv(ops: SwapOperationData[]): Ohlcv[] { return ohlcv; } -export function calculatePriceFromSwapOp(op: SwapOperationData): number { +export function calculateBasePriceFromSwapOp(op: SwapOperationData): number { if (!op || !op.offerAmount || !op.returnAmount) { return 0; } @@ -329,8 +345,8 @@ export function getSwapDirection(offerDenom: string, askDenom: string): SwapDire } const assetInfos = pair.asset_infos; // use quote denom as offer then its buy. Quote denom in pairs is the 2nd index in the array - if (assetInfos[0] === offerDenom) return "Sell"; - return "Buy"; + if (assetInfos[0] === askDenom) return "Buy"; + return "Sell"; } export function findPairIndexFromDenoms(offerDenom: string, askDenom: string): number { diff --git a/packages/oraidex-sync/src/pairs.ts b/packages/oraidex-sync/src/pairs.ts index 33758dd4..0226e824 100644 --- a/packages/oraidex-sync/src/pairs.ts +++ b/packages/oraidex-sync/src/pairs.ts @@ -97,3 +97,7 @@ export const pairsOnlyDenom = pairs.map((pair) => ({ })); export const uniqueInfos = extractUniqueAndFlatten(pairs); + +export const oraiUsdtPairOnlyDenom = pairsOnlyDenom.find( + (pair) => JSON.stringify(pair.asset_infos) === JSON.stringify([ORAI, usdtCw20Address]) +).asset_infos; diff --git a/packages/oraidex-sync/src/query.ts b/packages/oraidex-sync/src/query.ts index 886d79d6..5dae18fe 100644 --- a/packages/oraidex-sync/src/query.ts +++ b/packages/oraidex-sync/src/query.ts @@ -66,7 +66,7 @@ async function simulateSwapPrice(pairPath: AssetInfo[], router: OraiswapRouterRe if (operations.length === 0) return "0"; // error case. Will be handled by the caller function try { const data = await router.simulateSwapOperations({ - offerAmount: tenAmountInDecimalSix, + offerAmount: tenAmountInDecimalSix.toString(), operations }); return toDisplay(data.amount, 7).toString(); // since we simulate using 10 units, not 1. We use 10 because its a workaround for pools that are too small to simulate using 1 unit diff --git a/packages/oraidex-sync/src/test-db.ts b/packages/oraidex-sync/src/test-db.ts index 1db28ef4..b7d20c03 100644 --- a/packages/oraidex-sync/src/test-db.ts +++ b/packages/oraidex-sync/src/test-db.ts @@ -6,15 +6,18 @@ import { parseAssetInfoOnlyDenom } from "./helper"; import { simulateSwapPriceWithUsdt } from "./query"; import "dotenv/config"; +export function getDate24hBeforeNow(time: Date) { + const twentyFourHoursInMilliseconds = 24 * 60 * 60 * 1000; // 24 hours in milliseconds + const date24hBeforeNow = new Date(time.getTime() - twentyFourHoursInMilliseconds); + return date24hBeforeNow; +} + const start = async () => { - const duckdb = await DuckDb.create("oraidex-sync-data-staging"); + const duckdb = await DuckDb.create("oraidex-sync-data"); const tf = 86400; - const firstTokenResult = await duckdb.getOhlcvCandles( - "orai-orai12hzjxfh77wl572gdzct2fxv2arxcwh6gykc7qh", - 60, - 1688789160, - 1688796360 - ); + const now = new Date(); + const then = getDate24hBeforeNow(now); + const firstTokenResult = await duckdb.conn.all("select * from swap_ohlcv limit 5"); console.log(firstTokenResult); // let swapTokenMap = []; diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index 3614257e..e1edd24b 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -19,6 +19,7 @@ import { import { Log } from "@cosmjs/stargate/build/logs"; import { buildOhlcv, + calculatePriceByPool, concatDataToUniqueKey, getSwapDirection, groupByTime, @@ -128,21 +129,22 @@ function extractMsgProvideLiquidity( ): ProvideLiquidityOperationData | undefined { if ("provide_liquidity" in msg) { const assetInfos = msg.provide_liquidity.assets.map((asset) => asset.info); - let firstAsset = msg.provide_liquidity.assets[0]; - let secAsset = msg.provide_liquidity.assets[1]; + let baseAsset = msg.provide_liquidity.assets[0]; + let quoteAsset = msg.provide_liquidity.assets[1]; if (isAssetInfoPairReverse(assetInfos)) { - firstAsset = msg.provide_liquidity.assets[1]; - secAsset = msg.provide_liquidity.assets[0]; + baseAsset = msg.provide_liquidity.assets[1]; + quoteAsset = msg.provide_liquidity.assets[0]; } - const firstDenom = parseAssetInfoOnlyDenom(firstAsset.info); - const secDenom = parseAssetInfoOnlyDenom(secAsset.info); - const firstAmount = parseInt(firstAsset.amount); - const secAmount = parseInt(secAsset.amount); + const firstDenom = parseAssetInfoOnlyDenom(baseAsset.info); + const secDenom = parseAssetInfoOnlyDenom(quoteAsset.info); + const firstAmount = parseInt(baseAsset.amount); + const secAmount = parseInt(quoteAsset.amount); return { - firstTokenAmount: firstAmount, - firstTokenDenom: firstDenom, - firstTokenLp: firstAmount, + basePrice: calculatePriceByPool(BigInt(firstAmount), BigInt(secAmount)), + baseTokenAmount: firstAmount, + baseTokenDenom: firstDenom, + baseTokenReserve: firstAmount, opType: "provide", uniqueKey: concatDataToUniqueKey({ txheight: txData.txheight, @@ -151,9 +153,9 @@ function extractMsgProvideLiquidity( secondAmount: secAmount, secondDenom: secDenom }), - secondTokenAmount: secAmount, - secondTokenDenom: secDenom, - secondTokenLp: secAmount, + quoteTokenAmount: secAmount, + quoteTokenDenom: secDenom, + quoteTokenReserve: secAmount, timestamp: txData.timestamp, txCreator, txhash: txData.txhash, @@ -184,34 +186,38 @@ function extractMsgWithdrawLiquidity( if (!assetAttr) continue; const assets = parseWithdrawLiquidityAssets(assetAttr.value); // sanity check. only push data if can parse asset successfully - let firstAsset = assets[1]; - let secAsset = assets[3]; + let baseAssetAmount = parseInt(assets[0]); + let baseAsset = assets[1]; + let quoteAsset = assets[3]; + let quoteAssetAmount = parseInt(assets[2]); + // we only have one pair order. If the order is reversed then we also reverse the order if ( pairs.find( (pair) => JSON.stringify(pair.asset_infos.map((info) => parseAssetInfoOnlyDenom(info))) === - JSON.stringify([secAsset, firstAsset]) + JSON.stringify([quoteAsset, baseAsset]) ) ) { - firstAsset = assets[3]; - secAsset = assets[1]; + baseAsset = assets[3]; + quoteAsset = assets[1]; } if (assets.length !== 4) continue; withdrawData.push({ - firstTokenAmount: parseInt(assets[0]), - firstTokenDenom: assets[1], - firstTokenLp: parseInt(assets[0]), + basePrice: calculatePriceByPool(BigInt(baseAssetAmount), BigInt(quoteAssetAmount)), + baseTokenAmount: baseAssetAmount, + baseTokenDenom: assets[1], + baseTokenReserve: baseAssetAmount, opType: "withdraw", uniqueKey: concatDataToUniqueKey({ txheight: txData.txheight, - firstDenom: assets[1], - firstAmount: parseInt(assets[0]), - secondDenom: assets[3], - secondAmount: parseInt(assets[2]) + firstDenom: baseAsset, + firstAmount: baseAssetAmount, + secondDenom: quoteAsset, + secondAmount: quoteAssetAmount }), - secondTokenAmount: parseInt(assets[2]), - secondTokenDenom: assets[3], - secondTokenLp: parseInt(assets[2]), + quoteTokenAmount: quoteAssetAmount, + quoteTokenDenom: quoteAsset, + quoteTokenReserve: quoteAssetAmount, timestamp: txData.timestamp, txCreator, txhash: txData.txhash, diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 8d4143f3..04f9bd50 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -58,12 +58,13 @@ export type AccountTx = { export type LiquidityOpType = "provide" | "withdraw"; export type ProvideLiquidityOperationData = { - firstTokenAmount: number; - firstTokenDenom: string; // eg: orai, orai1234... - firstTokenLp: number | bigint; - secondTokenAmount: number; - secondTokenDenom: string; - secondTokenLp: number | bigint; + basePrice: number; + baseTokenAmount: number; + baseTokenDenom: string; // eg: orai, orai1234... + baseTokenReserve: number | bigint; + quoteTokenAmount: number; + quoteTokenDenom: string; + quoteTokenReserve: number | bigint; opType: LiquidityOpType; uniqueKey: string; // concat of first, second denom, amount, and timestamp => should be unique. unique key is used to override duplication only. txCreator: string; @@ -166,9 +167,17 @@ export interface Ohlcv { uniqueKey: string; // concat of timestamp, pair and volume. Only use to override potential duplication when inserting timestamp: number; pair: string; - volume: bigint; + volume: bigint; // base volume open: number; - close: number; + close: number; // base price low: number; high: number; } + +export type VolumeRange = { + time: string; + pair: string; + baseVolume: bigint; + quoteVolume: bigint; + basePrice: number; +}; diff --git a/packages/oraidex-sync/tests/db.spec.ts b/packages/oraidex-sync/tests/db.spec.ts index 95741564..3f14528d 100644 --- a/packages/oraidex-sync/tests/db.spec.ts +++ b/packages/oraidex-sync/tests/db.spec.ts @@ -172,15 +172,16 @@ describe("test-duckdb", () => { await expect( duckDb.insertLpOps([ { + basePrice: 1, txhash: "foo", timestamp: new Date().getTime() / 1000, - firstTokenAmount: "abcd" as any, - firstTokenLp: 0, - firstTokenDenom: "orai", + baseTokenAmount: "abcd" as any, + baseTokenReserve: 0, + baseTokenDenom: "orai", uniqueKey: "1", - secondTokenAmount: 2, - secondTokenLp: 0, - secondTokenDenom: "atom", + quoteTokenAmount: 2, + quoteTokenReserve: 0, + quoteTokenDenom: "atom", txCreator: "foobar", opType: "provide", txheight: 1 @@ -197,14 +198,15 @@ describe("test-duckdb", () => { const newDate = 1689610068000 / 1000; const data: ProvideLiquidityOperationData[] = [ { - firstTokenAmount: 1, - firstTokenDenom: "orai", - firstTokenLp: 0, + basePrice: 1, + baseTokenAmount: 1, + baseTokenDenom: "orai", + baseTokenReserve: 0, opType: "withdraw", uniqueKey: "2", - secondTokenAmount: 2, - secondTokenDenom: "atom", - secondTokenLp: 0, + quoteTokenAmount: 2, + quoteTokenDenom: "atom", + quoteTokenReserve: 0, timestamp: newDate, txCreator: "foobar", txhash: "foo", @@ -224,14 +226,15 @@ describe("test-duckdb", () => { const currentTimeStamp = Math.round(new Date().getTime() / 1000); let data: ProvideLiquidityOperationData[] = [ { - firstTokenAmount: 1, - firstTokenDenom: "orai", - firstTokenLp: 0, + basePrice: 1, + baseTokenAmount: 1, + baseTokenDenom: "orai", + baseTokenReserve: 0, opType: "withdraw", uniqueKey: "2", - secondTokenAmount: 2, - secondTokenDenom: "atom", - secondTokenLp: 0, + quoteTokenAmount: 2, + quoteTokenDenom: "atom", + quoteTokenReserve: 0, timestamp: currentTimeStamp, txCreator: "foobar", txhash: "foo", diff --git a/packages/oraidex-sync/tests/helper.spec.ts b/packages/oraidex-sync/tests/helper.spec.ts index 9216713d..3730715b 100644 --- a/packages/oraidex-sync/tests/helper.spec.ts +++ b/packages/oraidex-sync/tests/helper.spec.ts @@ -346,9 +346,10 @@ describe("test-helper", () => { ]); }); - it("test-calculatePriceByPool", () => { - const result = calculatePriceByPool(BigInt(10305560305234), BigInt(10205020305234), 0); - expect(result).toEqual(BigInt(9902432)); + it("test-calculatePriceByPool-ORAI/USDT-pool-when-1ORAI=2.74USDT", () => { + // base denom is ORAI, quote denom is USDT => base pool is ORAI, quote pool is USDT. + const result = calculatePriceByPool(BigInt(639997269712), BigInt(232967274783), 0, 10 ** 6); + expect(result.toString()).toEqual("2.747144"); }); it("test-collectAccumulateLpData-should-aggregate-ops-with-same-pairs", () => { @@ -370,12 +371,13 @@ describe("test-helper", () => { ]; const ops: ProvideLiquidityOperationData[] = [ { - firstTokenAmount: 1, - firstTokenDenom: ORAI, - secondTokenAmount: 1, - secondTokenDenom: usdtCw20Address, - firstTokenLp: 1, - secondTokenLp: 1, + basePrice: 1, + baseTokenAmount: 1, + baseTokenDenom: ORAI, + quoteTokenAmount: 1, + quoteTokenDenom: usdtCw20Address, + baseTokenReserve: 1, + quoteTokenReserve: 1, opType: "provide", uniqueKey: "1", timestamp: 1, @@ -384,12 +386,13 @@ describe("test-helper", () => { txheight: 1 }, { - firstTokenAmount: 1, - firstTokenDenom: ORAI, - secondTokenAmount: 1, - secondTokenDenom: usdtCw20Address, - firstTokenLp: 1, - secondTokenLp: 1, + basePrice: 1, + baseTokenAmount: 1, + baseTokenDenom: ORAI, + quoteTokenAmount: 1, + quoteTokenDenom: usdtCw20Address, + baseTokenReserve: 1, + quoteTokenReserve: 1, opType: "withdraw", uniqueKey: "2", timestamp: 1, @@ -398,12 +401,13 @@ describe("test-helper", () => { txheight: 1 }, { - firstTokenAmount: 1, - firstTokenDenom: ORAI, - secondTokenAmount: 1, - secondTokenDenom: atomIbcDenom, - firstTokenLp: 1, - secondTokenLp: 1, + basePrice: 1, + baseTokenAmount: 1, + baseTokenDenom: ORAI, + quoteTokenAmount: 1, + quoteTokenDenom: atomIbcDenom, + baseTokenReserve: 1, + quoteTokenReserve: 1, opType: "withdraw", uniqueKey: "3", timestamp: 1, @@ -414,12 +418,12 @@ describe("test-helper", () => { ]; collectAccumulateLpData(ops, poolResponses); - expect(ops[0].firstTokenLp.toString()).toEqual("2"); - expect(ops[0].secondTokenLp.toString()).toEqual("2"); - expect(ops[1].firstTokenLp.toString()).toEqual("1"); - expect(ops[1].secondTokenLp.toString()).toEqual("1"); - expect(ops[2].firstTokenLp.toString()).toEqual("3"); - expect(ops[2].secondTokenLp.toString()).toEqual("3"); + expect(ops[0].baseTokenReserve.toString()).toEqual("2"); + expect(ops[0].quoteTokenReserve.toString()).toEqual("2"); + expect(ops[1].baseTokenReserve.toString()).toEqual("1"); + expect(ops[1].quoteTokenReserve.toString()).toEqual("1"); + expect(ops[2].baseTokenReserve.toString()).toEqual("3"); + expect(ops[2].quoteTokenReserve.toString()).toEqual("3"); }); it("test-concatDataToUniqueKey-should-return-unique-key-in-correct-order-from-timestamp-to-first-to-second-amount-and-denom", () => { @@ -440,12 +444,13 @@ describe("test-helper", () => { it("test-remove-ops-duplication-should-remove-duplication-keys-before-inserting", () => { const ops: ProvideLiquidityOperationData[] = [ { - firstTokenAmount: 1, - firstTokenDenom: ORAI, - secondTokenAmount: 1, - secondTokenDenom: usdtCw20Address, - firstTokenLp: 1, - secondTokenLp: 1, + basePrice: 1, + baseTokenAmount: 1, + baseTokenDenom: ORAI, + quoteTokenAmount: 1, + quoteTokenDenom: usdtCw20Address, + baseTokenReserve: 1, + quoteTokenReserve: 1, opType: "provide", uniqueKey: "1", timestamp: 1, @@ -454,12 +459,13 @@ describe("test-helper", () => { txheight: 1 }, { - firstTokenAmount: 1, - firstTokenDenom: ORAI, - secondTokenAmount: 1, - secondTokenDenom: usdtCw20Address, - firstTokenLp: 1, - secondTokenLp: 1, + basePrice: 1, + baseTokenAmount: 1, + baseTokenDenom: ORAI, + quoteTokenAmount: 1, + quoteTokenDenom: usdtCw20Address, + baseTokenReserve: 1, + quoteTokenReserve: 1, opType: "withdraw", uniqueKey: "2", timestamp: 1, @@ -468,12 +474,13 @@ describe("test-helper", () => { txheight: 1 }, { - firstTokenAmount: 1, - firstTokenDenom: ORAI, - secondTokenAmount: 1, - secondTokenDenom: atomIbcDenom, - firstTokenLp: 1, - secondTokenLp: 1, + basePrice: 1, + baseTokenAmount: 1, + baseTokenDenom: ORAI, + quoteTokenAmount: 1, + quoteTokenDenom: atomIbcDenom, + baseTokenReserve: 1, + quoteTokenReserve: 1, opType: "withdraw", uniqueKey: "1", timestamp: 1, From ecc299240926284d7e08fdedb7b5e5cfa490b7ea Mon Sep 17 00:00:00 2001 From: trungbach Date: Thu, 3 Aug 2023 15:43:19 +0700 Subject: [PATCH 5/5] add: added api get candles & filter swapOps by direction --- packages/oraidex-server/src/index.ts | 12 +++++++++++- packages/oraidex-sync/src/db.ts | 7 +++++-- packages/oraidex-sync/src/helper.ts | 4 +++- packages/oraidex-sync/src/index.ts | 11 +++++++++++ packages/oraidex-sync/src/tx-parsing.ts | 1 + packages/oraidex-sync/src/types.ts | 7 +++++++ 6 files changed, 38 insertions(+), 4 deletions(-) diff --git a/packages/oraidex-server/src/index.ts b/packages/oraidex-server/src/index.ts index 0a824297..9d381c55 100644 --- a/packages/oraidex-server/src/index.ts +++ b/packages/oraidex-server/src/index.ts @@ -1,5 +1,5 @@ import * as dotenv from "dotenv"; -import express from "express"; +import express, { Request } from "express"; import { DuckDb, TickerInfo, @@ -18,6 +18,7 @@ import cors from "cors"; import { CosmWasmClient } from "@cosmjs/cosmwasm-stargate"; import { OraiswapRouterQueryClient } from "@oraichain/oraidex-contracts-sdk"; import { getDate24hBeforeNow, getSpecificDateBeforeNow, pairToString, parseSymbolsToTickerId } from "./helper"; +import { GetCandlesQuery } from "@oraichain/oraidex-sync"; dotenv.config(); @@ -209,6 +210,15 @@ app.get("/volume/v2/historical/chart", async (req, res) => { // } // }); +app.get("/v1/candles/", async (req: Request<{}, {}, {}, GetCandlesQuery>, res) => { + try { + const candles = await duckDb.getOhlcvCandles(req.query); + res.status(200).send(candles); + } catch (error) { + res.status(500).send(error.message); + } +}); + app.listen(port, hostname, async () => { // sync data for the service to read // console.dir(pairInfos, { depth: null }); diff --git a/packages/oraidex-sync/src/db.ts b/packages/oraidex-sync/src/db.ts index d437fd23..9d6ec51e 100644 --- a/packages/oraidex-sync/src/db.ts +++ b/packages/oraidex-sync/src/db.ts @@ -8,7 +8,8 @@ import { TotalLiquidity, VolumeData, VolumeRange, - WithdrawLiquidityOperationData + WithdrawLiquidityOperationData, + GetCandlesQuery } from "./types"; import fs, { rename } from "fs"; import { isoToTimestampNumber, renameKey, replaceAllNonAlphaBetChar, toObject } from "./helper"; @@ -306,7 +307,9 @@ export class DuckDb { await this.insertBulkData(ohlcv, "swap_ohlcv"); } - async getOhlcvCandles(pair: string, tf: number, startTime: number, endTime: number): Promise { + async getOhlcvCandles(query: GetCandlesQuery): Promise { + const { pair, tf, startTime, endTime } = query; + // tf should be in seconds const result = await this.conn.all( ` diff --git a/packages/oraidex-sync/src/helper.ts b/packages/oraidex-sync/src/helper.ts index adbe2cac..84ff99f2 100644 --- a/packages/oraidex-sync/src/helper.ts +++ b/packages/oraidex-sync/src/helper.ts @@ -341,7 +341,9 @@ export function getSwapDirection(offerDenom: string, askDenom: string): SwapDire return pair.asset_infos.some((info) => info === offerDenom) && pair.asset_infos.some((info) => info === askDenom); }); if (!pair) { - throw new Error("Cannot find asset infos in list of pairs"); + console.error("Cannot find asset infos in list of pairs"); + return; + // throw new Error("Cannot find asset infos in list of pairs"); } const assetInfos = pair.asset_infos; // use quote denom as offer then its buy. Quote denom in pairs is the 2nd index in the array diff --git a/packages/oraidex-sync/src/index.ts b/packages/oraidex-sync/src/index.ts index d7e0f4ce..b5e6897b 100644 --- a/packages/oraidex-sync/src/index.ts +++ b/packages/oraidex-sync/src/index.ts @@ -187,6 +187,17 @@ class OraiDexSync { } } +// async function initSync() { +// const duckDb = await DuckDb.create(process.env.DUCKDB_PROD_FILENAME || "oraidex-sync-data"); +// const oraidexSync = await OraiDexSync.create( +// duckDb, +// process.env.RPC_URL || "https://rpc.orai.io", +// process.env as any +// ); +// oraidexSync.sync(); +// } + +// initSync(); export { OraiDexSync }; export * from "./types"; diff --git a/packages/oraidex-sync/src/tx-parsing.ts b/packages/oraidex-sync/src/tx-parsing.ts index e1edd24b..a9e0248c 100644 --- a/packages/oraidex-sync/src/tx-parsing.ts +++ b/packages/oraidex-sync/src/tx-parsing.ts @@ -283,6 +283,7 @@ function parseTxs(txs: Tx[]): TxAnlysisResult { accountTxs.push({ txhash: basicTxData.txhash, accountAddress: sender }); } } + swapOpsData = swapOpsData.filter((i) => i.direction); swapOpsData = removeOpsDuplication(swapOpsData) as SwapOperationData[]; return { // transactions: txs, diff --git a/packages/oraidex-sync/src/types.ts b/packages/oraidex-sync/src/types.ts index 04f9bd50..d6e3cfd4 100644 --- a/packages/oraidex-sync/src/types.ts +++ b/packages/oraidex-sync/src/types.ts @@ -181,3 +181,10 @@ export type VolumeRange = { quoteVolume: bigint; basePrice: number; }; + +export type GetCandlesQuery = { + pair: string; + tf: number; + startTime: number; + endTime: number; +};