Merge pull request #8 from oraichain/feat/oraidex-info

Feat/oraidex info

packages/contracts-build/package.json

@@ -8,6 +8,6 @@
     "data/"
   ],
   "dependencies": {
-    "@oraichain/oraidex-contracts-sdk": "^1.0.10"
+    "@oraichain/oraidex-contracts-sdk": "^1.0.18"
   }
 }

packages/oraidex-server/package.json

@@ -14,7 +14,6 @@
     "@cosmjs/stargate": "^0.31.0",
     "@oraichain/common-contracts-sdk": "1.0.13",
     "@cosmjs/tendermint-rpc": "^0.31.0",
-    "@oraichain/cosmos-rpc-sync": "^1.0.5",
     "@oraichain/oraidex-sync": "1.0.0",
     "cors": "^2.8.5",
     "express": "^4.18.2"

packages/oraidex-server/src/helper.ts

@@ -7,3 +7,23 @@ export function getDate24hBeforeNow(time: Date) {
   const date24hBeforeNow = new Date(time.getTime() - twentyFourHoursInMilliseconds);
   return date24hBeforeNow;
 }
+
+/**
+ *
+ * @param time
+ * @param tf in seconds
+ * @returns
+ */
+export function getSpecificDateBeforeNow(time: Date, tf: number) {
+  const timeInMs = tf * 1000; // 24 hours in milliseconds
+  const dateBeforeNow = new Date(time.getTime() - timeInMs);
+  return dateBeforeNow;
+}
+
+export function calculateBasePriceFromTickerVolume(baseVolume: string, targetVolume: string): number {
+  return parseFloat(targetVolume) / parseFloat(baseVolume);
+}
+
+export function pairToString(pair: string[]): string {
+  return `${pair[0]}-${pair[1]}`;
+}

packages/oraidex-server/src/index.ts

@@ -1,5 +1,5 @@
 import * as dotenv from "dotenv";
-import express from "express";
+import express, { Request } from "express";
 import {
   DuckDb,
   TickerInfo,
@@ -9,16 +9,16 @@ import {
   toDisplay,
   OraiDexSync,
   simulateSwapPrice,
-  getPoolInfos,
-  calculatePrefixSum,
-  uniqueInfos,
-  simulateSwapPriceWithUsdt
+  pairsOnlyDenom,
+  VolumeRange,
+  oraiUsdtPairOnlyDenom,
+  ORAI
 } from "@oraichain/oraidex-sync";
 import cors from "cors";
 import { CosmWasmClient } from "@cosmjs/cosmwasm-stargate";
-import { OraiswapRouterQueryClient, PairInfo } from "@oraichain/oraidex-contracts-sdk";
-import { getDate24hBeforeNow, parseSymbolsToTickerId } from "./helper";
-import { MulticallQueryClient } from "@oraichain/common-contracts-sdk";
+import { OraiswapRouterQueryClient } from "@oraichain/oraidex-contracts-sdk";
+import { getDate24hBeforeNow, getSpecificDateBeforeNow, pairToString, parseSymbolsToTickerId } from "./helper";
+import { GetCandlesQuery } from "@oraichain/oraidex-sync";
 
 dotenv.config();
 
@@ -59,6 +59,8 @@ app.get("/tickers", async (req, res) => {
       process.env.ROUTER_CONTRACT_ADDRESS || "orai1j0r67r9k8t34pnhy00x3ftuxuwg0r6r4p8p6rrc8az0ednzr8y9s3sj2sf"
     );
     const pairInfos = await duckDb.queryPairInfos();
+    const latestTimestamp = endTime ? parseInt(endTime as string) : await duckDb.queryLatestTimestampSwapOps();
+    const then = getDate24hBeforeNow(new Date(latestTimestamp * 1000)).getTime() / 1000;
     const data: TickerInfo[] = (
       await Promise.allSettled(
         pairs.map(async (pair) => {
@@ -68,8 +70,6 @@ app.get("/tickers", async (req, res) => {
           // const { baseIndex, targetIndex, target } = findUsdOraiInPair(pair.asset_infos);
           const baseIndex = 0;
           const targetIndex = 1;
-          const latestTimestamp = endTime ? parseInt(endTime as string) : await duckDb.queryLatestTimestampSwapOps();
-          const then = getDate24hBeforeNow(new Date(latestTimestamp * 1000)).getTime() / 1000;
           console.log(latestTimestamp, then);
           const baseInfo = parseAssetInfoOnlyDenom(pair.asset_infos[baseIndex]);
           const targetInfo = parseAssetInfoOnlyDenom(pair.asset_infos[targetIndex]);
@@ -110,59 +110,47 @@ app.get("/tickers", async (req, res) => {
 // TODO: refactor this and add unit tests
 app.get("/volume/v2/historical/chart", async (req, res) => {
   const { startTime, endTime, tf } = req.query;
-  const timeFrame = parseInt(tf as string);
-  const volumeInfos = await duckDb.pivotVolumeRange(parseInt(startTime as string), parseInt(endTime as string));
-  const cosmwasmClient = await CosmWasmClient.connect(process.env.RPC_URL);
-  let finalArray = [];
-  let prices;
-  let heightCount = 0;
-  for (let i = 0; i < volumeInfos.length; i++) {
-    const volInfo = volumeInfos[i];
-    cosmwasmClient.setQueryClientWithHeight(volInfo.txheight);
-    const router = new OraiswapRouterQueryClient(
-      cosmwasmClient,
-      process.env.ROUTER_CONTRACT_ADDRESS || "orai1j0r67r9k8t34pnhy00x3ftuxuwg0r6r4p8p6rrc8az0ednzr8y9s3sj2sf"
-    );
-    if (heightCount % 1000 === 0) {
-      // prevent simulating too many times. TODO: calculate this using pool data from
-      prices = (await Promise.all(uniqueInfos.map((info) => simulateSwapPriceWithUsdt(info, router))))
-        .map((price) => ({ ...price, info: parseAssetInfoOnlyDenom(price.info) }))
-        .reduce((acc, cur) => {
-          acc[cur.info] = parseFloat(cur.amount);
-          return acc;
-        }, {});
-    }
-    let tempData = {};
-    for (const key in volInfo) {
-      if (key === "timestamp" || key === "txheight") continue;
-      if (Object.keys(tempData).includes("volume_price")) {
-        tempData["volume_price"] += volInfo[key] * prices[key];
-      } else {
-        tempData["timestamp"] = volInfo["timestamp"];
-        tempData["volume_price"] = 0;
-      }
-    }
-    const indexOf = finalArray.findIndex((data) => data.timestamp === tempData["timestamp"]);
-    if (indexOf === -1) finalArray.push(tempData);
-    else {
-      finalArray[indexOf] = {
-        ...finalArray[indexOf],
-        volume_price: finalArray[indexOf].volume_price + tempData["volume_price"]
-      };
+  const timeFrame = tf ? parseInt(tf as string) : 60;
+  const latestTimestamp = endTime ? parseInt(endTime as string) : await duckDb.queryLatestTimestampSwapOps();
+  const then = startTime
+    ? parseInt(startTime as string)
+    : getSpecificDateBeforeNow(new Date(latestTimestamp * 1000), 259200).getTime() / 1000;
+  const volumeInfos = await Promise.all(
+    pairsOnlyDenom.map((pair) => {
+      return duckDb.getVolumeRange(timeFrame, then, latestTimestamp, pairToString(pair.asset_infos));
+    })
+  );
+  // console.log("volume infos: ", volumeInfos);
+  let volumeRanges: { [time: string]: VolumeRange[] } = {};
+  for (let volumePair of volumeInfos) {
+    for (let volume of volumePair) {
+      if (!volumeRanges[volume.time]) volumeRanges[volume.time] = [{ ...volume }];
+      else volumeRanges[volume.time].push({ ...volume });
     }
-    heightCount++;
   }
-  let finalFinalArray = [];
-  for (let data of finalArray) {
-    let time = Math.floor(data.timestamp / timeFrame);
-    let index = finalFinalArray.findIndex((data) => data.timestamp === time);
-    if (index === -1) {
-      finalFinalArray.push({ timestamp: time, volume_price: data.volume_price });
-    } else {
-      finalFinalArray[index].volume_price += data.volume_price;
+  let result = [];
+  for (let [time, volumeData] of Object.entries(volumeRanges)) {
+    const oraiUsdtVolumeData = volumeData.find((data) => data.pair === pairToString(oraiUsdtPairOnlyDenom));
+    if (!oraiUsdtVolumeData) {
+      res.status(500).send("Cannot find ORAI_USDT volume data in the volume list");
     }
+    const totalVolumePrice = volumeData.reduce((acc, volData) => {
+      // console.log("base price in usdt: ", basePriceInUsdt);
+      // if base denom is orai then we calculate vol using quote vol
+      let volumePrice = 0;
+      if (volData.pair.split("-")[0] === ORAI) {
+        volumePrice = oraiUsdtVolumeData.basePrice * toDisplay(BigInt(volData.baseVolume));
+      } else if (volData.pair.split("-")[1] === ORAI) {
+        volumePrice = oraiUsdtVolumeData.basePrice * toDisplay(BigInt(volData.quoteVolume));
+      } else {
+        return acc; // skip for now cuz dont know how to calculate price if not paired if with ORAI
+      }
+      // volume price is calculated based on the base currency & quote volume
+      return acc + volumePrice;
+    }, 0);
+    result.push({ time, value: totalVolumePrice });
   }
-  res.status(200).send(finalFinalArray);
+  res.status(200).send(result);
 });
 
 // app.get("/liquidity/v2/historical/chart", async (req, res) => {
@@ -222,6 +210,15 @@ app.get("/volume/v2/historical/chart", async (req, res) => {
 //   }
 // });
 
+app.get("/v1/candles/", async (req: Request<{}, {}, {}, GetCandlesQuery>, res) => {
+  try {
+    const candles = await duckDb.getOhlcvCandles(req.query);
+    res.status(200).send(candles);
+  } catch (error) {
+    res.status(500).send(error.message);
+  }
+});
+
 app.listen(port, hostname, async () => {
   // sync data for the service to read
   // console.dir(pairInfos, { depth: null });

packages/oraidex-sync/package.json

@@ -12,10 +12,11 @@
   "dependencies": {
     "@cosmjs/tendermint-rpc": "^0.31.0",
     "@oraichain/common-contracts-sdk": "1.0.13",
-    "@oraichain/cosmos-rpc-sync": "^1.0.6",
+    "@oraichain/cosmos-rpc-sync": "^1.0.7",
     "@oraichain/oraidex-contracts-sdk": "^1.0.13",
     "duckdb-async": "^0.8.1",
-    "apache-arrow": "^12.0.1"
+    "apache-arrow": "^12.0.1",
+    "lodash": "^4.17.21"
   },
   "devDependencies": {
     "@types/lodash": "^4.14.195"

packages/oraidex-sync/src/constants.ts

@@ -10,6 +10,6 @@ export const scOraiCw20Address = "orai1065qe48g7aemju045aeyprflytemx7kecxkf5m7u5
 export const usdcCw20Address = "orai15un8msx3n5zf9ahlxmfeqd2kwa5wm0nrpxer304m9nd5q6qq0g6sku5pdd";
 export const atomIbcDenom = "ibc/A2E2EEC9057A4A1C2C0A6A4C78B0239118DF5F278830F50B4A6BDD7A66506B78";
 export const osmosisIbcDenom = "ibc/9C4DCD21B48231D0BC2AC3D1B74A864746B37E4292694C93C617324250D002FC";
-export const tenAmountInDecimalSix = "10000000";
+export const tenAmountInDecimalSix = 10000000;
 export const truncDecimals = 6;
 export const atomic = 10 ** truncDecimals;

packages/oraidex-sync/src/db.ts

@@ -1,13 +1,15 @@
 import { Database, Connection } from "duckdb-async";
 import {
+  Ohlcv,
   PairInfoData,
   PriceInfo,
   SwapOperationData,
   TokenVolumeData,
   TotalLiquidity,
   VolumeData,
-  VolumeInfo,
-  WithdrawLiquidityOperationData
+  VolumeRange,
+  WithdrawLiquidityOperationData,
+  GetCandlesQuery
 } from "./types";
 import fs, { rename } from "fs";
 import { isoToTimestampNumber, renameKey, replaceAllNonAlphaBetChar, toObject } from "./helper";
@@ -54,10 +56,17 @@ export class DuckDb {
 
   // swap operation table handling
   async createSwapOpsTable() {
+    try {
+      await this.conn.all("select enum_range(NULL::directionType);");
+    } catch (error) {
+      // if error it means the enum does not exist => create new
+      await this.conn.exec("CREATE TYPE directionType AS ENUM ('Buy','Sell');");
+    }
     await this.conn.exec(
       `CREATE TABLE IF NOT EXISTS swap_ops_data (
         askDenom VARCHAR, 
         commissionAmount UBIGINT,
+        direction directionType,
         offerAmount UBIGINT,
         offerDenom VARCHAR, 
         uniqueKey VARCHAR UNIQUE,
@@ -84,14 +93,15 @@ export class DuckDb {
     }
     await this.conn.exec(
       `CREATE TABLE IF NOT EXISTS lp_ops_data (
-        firstTokenAmount UBIGINT, 
-        firstTokenDenom VARCHAR, 
-        firstTokenLp UBIGINT, 
+        basePrice double,
+        baseTokenAmount UBIGINT, 
+        baseTokenDenom VARCHAR, 
+        baseTokenReserve UBIGINT, 
         opType LPOPTYPE, 
         uniqueKey VARCHAR UNIQUE,
-        secondTokenAmount UBIGINT, 
-        secondTokenDenom VARCHAR, 
-        secondTokenLp UBIGINT,
+        quoteTokenAmount UBIGINT, 
+        quoteTokenDenom VARCHAR, 
+        quoteTokenReserve UBIGINT,
         timestamp UINTEGER,
         txCreator VARCHAR, 
         txhash VARCHAR,
@@ -258,7 +268,7 @@ export class DuckDb {
       `with pivot_lp_ops as (
         pivot lp_ops_data
         on opType
-        using sum(firstTokenAmount + secondTokenAmount) as liquidity )
+        using sum(baseTokenAmount + quoteTokenAmount) as liquidity )
         SELECT (timestamp // ?) as time,
         sum(COALESCE(provide_liquidity,0) - COALESCE(withdraw_liquidity, 0)) as liquidity,
         any_value(txheight) as height
@@ -278,39 +288,78 @@ export class DuckDb {
     return result as TotalLiquidity[];
   }
 
-  async createVolumeInfo() {
+  async createSwapOhlcv() {
     await this.conn.exec(
-      `CREATE TABLE IF NOT EXISTS volume_info (
-        denom VARCHAR,
-        timestamp UINTEGER,
-        txheight UINTEGER,
-        price DOUBLE,
-        volume UBIGINT)
+      `CREATE TABLE IF NOT EXISTS swap_ohlcv (
+        uniqueKey varchar UNIQUE,
+        timestamp uinteger,
+        pair varchar,
+        volume ubigint,
+        open double,
+        close double,
+        low double,
+        high double)
         `
     );
   }
 
-  async insertVolumeInfo(volumeInfos: VolumeInfo[]) {
-    await this.insertBulkData(volumeInfos, "volume_info");
+  async insertOhlcv(ohlcv: Ohlcv[]) {
+    await this.insertBulkData(ohlcv, "swap_ohlcv");
   }
 
-  async pivotVolumeRange(startTime: number, endTime: number) {
-    let volumeInfos = await this.conn.all(
+  async getOhlcvCandles(query: GetCandlesQuery): Promise<Ohlcv[]> {
+    const { pair, tf, startTime, endTime } = query;
+
+    // tf should be in seconds
+    const result = await this.conn.all(
       `
-      pivot (select * from volume_info
-      where timestamp >= ${startTime}
-      and timestamp <= ${endTime} 
-      order by timestamp) 
-      on denom 
-      using sum(volume) 
-      group by timestamp, txheight 
-      order by timestamp`
+        SELECT timestamp // ? as time,
+                sum(volume) as volume,
+                first(open) as open,
+                last(close) as close,
+                min(low) as low,
+                max(high) as high
+        FROM swap_ohlcv
+        WHERE pair = ? AND timestamp >= ? AND timestamp <= ?
+        GROUP BY time
+        ORDER BY time
+    `,
+      +tf,
+      pair,
+      startTime,
+      endTime
     );
-    for (let volInfo of volumeInfos) {
-      for (const key in volInfo) {
-        if (volInfo[key] === null) volInfo[key] = 0;
-      }
-    }
-    return volumeInfos;
+
+    // get second
+    result.forEach((item) => {
+      item.time *= tf;
+    });
+    return result as Ohlcv[];
+  }
+
+  async getVolumeRange(tf: number, startTime: number, endTime: number, pair: string): Promise<VolumeRange[]> {
+    const result = await this.conn.all(
+      `
+      SELECT timestamp // ? as time,
+        any_value(pair) as pair,
+        sum(volume) as baseVolume,
+        cast(sum(close * volume) as UBIGINT) as quoteVolume,
+        avg(close) as basePrice,
+        FROM swap_ohlcv
+        WHERE timestamp >= ? 
+        AND timestamp <= ?
+        and pair = ?
+        GROUP BY time
+        ORDER BY time
+      `,
+      tf,
+      startTime,
+      endTime,
+      pair
+    );
+    return result.map((res) => ({
+      ...res,
+      time: new Date(res.time * tf * 1000).toISOString()
+    })) as VolumeRange[];
   }
 }