From fe6c1b1c45093231901cd52b7759943a8af2cd65 Mon Sep 17 00:00:00 2001 From: siegfriedbz Date: Thu, 1 Aug 2024 11:10:59 +0800 Subject: [PATCH 1/9] add script to fetch all users --- scripts/fetchUsers.js | 123 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 scripts/fetchUsers.js diff --git a/scripts/fetchUsers.js b/scripts/fetchUsers.js new file mode 100644 index 00000000..c3e48488 --- /dev/null +++ b/scripts/fetchUsers.js @@ -0,0 +1,123 @@ +const fetchUsers = async () => { + try { + // fetch all pairs + const pairsList = await fetchAllPairs(); + + // Aggregate all pairs info + const pairsInfo = pairsList.map((pair) => ({ + pairAddress: pair.address, + lastOrderId: pair.lastOrderId, + })); + + // fetch all orders for all pairs + const allOrdersByPair = await Promise.all( + pairsInfo.map((pair) => { + const { pairAddress, lastOrderId } = pair; + const orderIds = Array.from({ length: lastOrderId }, (_, i) => i + 1); // [1, ..., lastOrderId] + + return fetchOrdersByPair(pairAddress, orderIds); + }) + ); + + // Process orders to count wallet addresses + const usersFreqCounter = allOrdersByPair.reduce((acc, curr) => { + if (!curr?.orders?.length) return acc; + + // process all orders for each pair + const orders = curr.orders; + + for (const order of orders) { + const radixWalletAddress = order?.settlementAccount; // account address for this order + + if (!radixWalletAddress) continue; + else { + acc[radixWalletAddress] = (acc[radixWalletAddress] || 0) + 1; + } + } + return acc; + }, {}); + + // get total unique users count + const totalUsersCount = Object.values(usersFreqCounter).reduce( + (acc, curr) => { + return acc + curr; + }, + 0 + ); + + return { usersFreqCounter, totalUsersCount }; + } catch (error) { + console.error("fetchUsers -> error", error); + return { usersFreqCounter: {}, totalUsersCount: 0 }; + } +}; + +export default fetchUsers; + +//** Helpers */ +// fetch all pairs +const fetchAllPairs = async () => { + try { + const allPairsResponse = await fetch( + "https://api.alphadex.net/v0/alphadex/pairs", + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + } + ); + + if (!allPairsResponse.ok) { + throw new Error("Error while fetching pairs"); + } + + const allPairsData = await allPairsResponse.json(); + const pairsList = allPairsData?.pairs || []; + + return pairsList; + } catch (error) { + console.error("fetchAllPairs -> error", error); + return []; + } +}; + +// fetch orders +const fetchOrdersByPair = async (pairAddress, orderIds) => { + try { + const allOrders = []; + const chunks = getChunkArray(orderIds, 99); + + // fetch all orders by batches of 99 orders + for (const chunkOrderIds of chunks) { + const response = await fetch("https://api.alphadex.net/v0/pair/orders", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ pairAddress, orderIds: chunkOrderIds }), + }); + + if (!response.ok) { + throw new Error("Error while fetching orders by pair"); + } + + const data = await response.json(); + allOrders.push(...data.orders); + } + + return { orders: allOrders }; + } catch (error) { + console.error("fetchOrdersByPair -> error", error); + return { orders: [] }; + } +}; + +const getChunkArray = (array, size) => { + const chunkArray = []; + for (let i = 0; i < array.length; i += size) { + chunkArray.push(array.slice(i, i + size)); + } + + return chunkArray; +}; From b9ca3f1b287bcf64a481706b8e809e8b9d3f93df Mon Sep 17 00:00:00 2001 From: dcts Date: Thu, 1 Aug 2024 15:01:43 +0200 Subject: [PATCH 2/9] simplify fetchUsers and rename variables --- scripts/fetchUsers.js | 81 ++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 52 deletions(-) diff --git a/scripts/fetchUsers.js b/scripts/fetchUsers.js index c3e48488..f80815eb 100644 --- a/scripts/fetchUsers.js +++ b/scripts/fetchUsers.js @@ -1,58 +1,35 @@ const fetchUsers = async () => { - try { - // fetch all pairs - const pairsList = await fetchAllPairs(); - - // Aggregate all pairs info - const pairsInfo = pairsList.map((pair) => ({ - pairAddress: pair.address, - lastOrderId: pair.lastOrderId, - })); - - // fetch all orders for all pairs - const allOrdersByPair = await Promise.all( - pairsInfo.map((pair) => { - const { pairAddress, lastOrderId } = pair; - const orderIds = Array.from({ length: lastOrderId }, (_, i) => i + 1); // [1, ..., lastOrderId] - - return fetchOrdersByPair(pairAddress, orderIds); - }) - ); - - // Process orders to count wallet addresses - const usersFreqCounter = allOrdersByPair.reduce((acc, curr) => { - if (!curr?.orders?.length) return acc; - - // process all orders for each pair - const orders = curr.orders; - - for (const order of orders) { - const radixWalletAddress = order?.settlementAccount; // account address for this order - - if (!radixWalletAddress) continue; - else { - acc[radixWalletAddress] = (acc[radixWalletAddress] || 0) + 1; - } - } - return acc; - }, {}); - - // get total unique users count - const totalUsersCount = Object.values(usersFreqCounter).reduce( - (acc, curr) => { - return acc + curr; - }, - 0 - ); - - return { usersFreqCounter, totalUsersCount }; - } catch (error) { - console.error("fetchUsers -> error", error); - return { usersFreqCounter: {}, totalUsersCount: 0 }; + // fetch all pairs + const pairsList = await fetchAllPairs(); + + // fetch all orders for all pairs + const allOrders = await Promise.all( + pairsList.map((pair) => { + const { address, lastOrderId } = pair; + const orderIds = Array.from({ length: lastOrderId }, (_, i) => i + 1); // [1, ..., lastOrderId] + + return fetchOrdersByPair(address, orderIds); + }) + ).flat(); + + // Process orders to count wallet addresses + const usersDict = {}; + for (let i = 0; i < allOrders.length; i++) { + const radixWalletAddress = order[i]?.settlementAccount; // account address for this order + if (!radixWalletAddress) { + continue; + } + usersDict[radixWalletAddress] = usersDict[radixWalletAddress] + ? usersDict[radixWalletAddress] + 1 + : 1; } -}; -export default fetchUsers; + return { + usersDict: usersDict, + totalUsers: Object.keys(usersDict).length, + totalOrders: Object.values(usersDict).reduce((a, b) => a + b), + }; +}; //** Helpers */ // fetch all pairs From 0a354708574a26a9720ce50ca8c35c1b27074371 Mon Sep 17 00:00:00 2001 From: dcts Date: Thu, 1 Aug 2024 15:04:52 +0200 Subject: [PATCH 3/9] simplify fetchOrdersByPair --- scripts/fetchUsers.js | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/scripts/fetchUsers.js b/scripts/fetchUsers.js index f80815eb..f6bae76d 100644 --- a/scripts/fetchUsers.js +++ b/scripts/fetchUsers.js @@ -61,12 +61,13 @@ const fetchAllPairs = async () => { // fetch orders const fetchOrdersByPair = async (pairAddress, orderIds) => { - try { - const allOrders = []; - const chunks = getChunkArray(orderIds, 99); + const allOrders = []; + const chunks = getChunkArray(orderIds, 99); - // fetch all orders by batches of 99 orders - for (const chunkOrderIds of chunks) { + // fetch all orders by batches of 99 orders + for (const chunkOrderIds of chunks) { + try { + console.log("Fetching batch for " + pairAddress); const response = await fetch("https://api.alphadex.net/v0/pair/orders", { method: "POST", headers: { @@ -75,19 +76,15 @@ const fetchOrdersByPair = async (pairAddress, orderIds) => { body: JSON.stringify({ pairAddress, orderIds: chunkOrderIds }), }); - if (!response.ok) { - throw new Error("Error while fetching orders by pair"); - } - const data = await response.json(); allOrders.push(...data.orders); - } - return { orders: allOrders }; - } catch (error) { - console.error("fetchOrdersByPair -> error", error); - return { orders: [] }; + } catch (err) { + console.log(err); + } } + + return { orders: allOrders }; }; const getChunkArray = (array, size) => { From 521f84bb73e5dbd386965871b7462cb9798cda5e Mon Sep 17 00:00:00 2001 From: dcts Date: Thu, 1 Aug 2024 15:07:27 +0200 Subject: [PATCH 4/9] make script runnable via > dexter-dapp@0.1.0 fetchUsers > node ./scripts/fetchUsers.js Fetching batch for component_rdx1cp824exl9mscrd7mcrr2dkze7uxfv3m6pr79n55da2jmkvtjm5wdvy Fetching batch for component_rdx1cpqtwr4pnv6sypkc2jcsw5e2xdwzmqm93t7yssdeqalz5h93c0v9fw Fetching batch for component_rdx1cpax5ncv7dejpn4l587umy4cwt2a3g8q0klq3g7wc2r3nx2dhrqmcr Fetching batch for component_rdx1crc2u544d0ys7nxksemv5ugq2ckpptg0kwlv89nfxdpvrv3zlejpa6 Fetching batch for component_rdx1crn2p6cqd4tewzqlgmct29rkphak6p9xlqf28eesr3z3ncg7y9k8gt Fetching batch for component_rdx1czd0fxvw4vl3rnvfsk556cqhxgx3psd96tnjwqwng22njnpde2aqka Fetching batch for component_rdx1czvg8gu5xsugwzhqr9prknyx4c6khntvnxert4fghs62zygjj9xvhl Fetching batch for component_rdx1cpl5hhxnwccrdfmrm9rgrt2ud08r5tf385ntemnsdv3wnqjlh3g6s7 Fetching batch for component_rdx1cqzv2v63gaexzf28g8slvr7y0xtdhzjetckqj0n0vwk9syk3klxtv2 Fetching batch for component_rdx1crmh8jc9pd3wcfm433at2fle62ajshgzk6d9n8nchmkxhjfsuc4zq6 Fetching batch for component_rdx1cqzyzxukhtwjdmppcf8zfw8nkakf2pu0kr7zr3qukewnrwg9ysqrhm Fetching batch for component_rdx1crr8jxrp8cn08c0x07hs2dhqathhsyx8z943qr20647tz33zm39dkx Fetching batch for component_rdx1czfn7n8z4yaeytezyyw8jxavejlr89d9dm27f87l0he9klzc3u3s9c Fetching batch for component_rdx1cr4urwgxap6xvyjlnudg08sk9t2gsdqrxh9mvetxs8kqlcyxh07qd3 Fetching batch for component_rdx1cz8pmscz5ttfzg9z8e6yxnx4acrxqqfwzrjyf2y44rjrn03hzsz9hm Fetching batch for component_rdx1crwnhghzrvtuldy7ulvl33rnkvq77e3v3y8vtk5ccm8tt9m56fvwaf Fetching batch for component_rdx1cqxrrfy22ngevqnx4aykrmx5wsu6l0ryaqm33sv9kx6sqzsrcdmqpm Fetching batch for component_rdx1cz9vzy0m3wj4kdsvuqtcptl9d2mkrjvv2ahr2x7mgr2nw9nvfc968g Fetching batch for component_rdx1czgjmu4ed5hp0vn97ngsf6mevq9tl0v8rrh2yq0f4dnpndggk7j9pu Fetching batch for component_rdx1cpttp5d5fc4lh95mmsnptt0xam2kgqt763za5tzx4xs2ycv9ec28wr Fetching batch for component_rdx1crzqxsshnx4mk2474vy9ans6p8v0njph6hvfrt025c7uq8lwf9h0w9 Fetching batch for component_rdx1crdxwcl5je8n20rrf8x26f5hddd8v2j32psza7fnwtrdn3wpsf333y Fetching batch for component_rdx1cqxehwlt2p05uqs96dw388tws325q39hr84uapsqayw46q0fg6l0wa Fetching batch for component_rdx1cpzh5k94hetqf5uc64m6ymh5ejkpx4l6539pqjj2xx3zdfat7mjwrj Fetching batch for component_rdx1crht8eesn5cxr9a9apq9rk0g86knwgu3q3at7j4lhefgzl29kxgtz5 --- package.json | 3 ++- scripts/fetchUsers.js | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 58767fa3..27d70888 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "test": "vitest", "test-integration": "playwright test", "test-all": "npm run lint && npm run test && npm run test-integration", - "copy-pr": "node ./scripts/copy-pr.js" + "copy-pr": "node ./scripts/copy-pr.js", + "fetchUsers": "node ./scripts/fetchUsers.js" }, "dependencies": { "@mdx-js/loader": "^3.0.0", diff --git a/scripts/fetchUsers.js b/scripts/fetchUsers.js index f6bae76d..6807645f 100644 --- a/scripts/fetchUsers.js +++ b/scripts/fetchUsers.js @@ -95,3 +95,10 @@ const getChunkArray = (array, size) => { return chunkArray; }; + + +// RUN SCRIPT +(async () => { + const result = await fetchUsers(); + console.log(result) +})(); \ No newline at end of file From e9e5aa3fbd1488bc0685179ac40bf2565f22a6d1 Mon Sep 17 00:00:00 2001 From: dcts Date: Thu, 1 Aug 2024 15:18:22 +0200 Subject: [PATCH 5/9] fix some minor issues to make script work again --- scripts/fetchUsers.js | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/fetchUsers.js b/scripts/fetchUsers.js index 6807645f..b33e0a20 100644 --- a/scripts/fetchUsers.js +++ b/scripts/fetchUsers.js @@ -3,31 +3,33 @@ const fetchUsers = async () => { const pairsList = await fetchAllPairs(); // fetch all orders for all pairs - const allOrders = await Promise.all( + const allOrdersByPairs = await Promise.all( pairsList.map((pair) => { const { address, lastOrderId } = pair; const orderIds = Array.from({ length: lastOrderId }, (_, i) => i + 1); // [1, ..., lastOrderId] return fetchOrdersByPair(address, orderIds); }) - ).flat(); + ); + const allOrders = allOrdersByPairs.flat(); // Process orders to count wallet addresses const usersDict = {}; for (let i = 0; i < allOrders.length; i++) { - const radixWalletAddress = order[i]?.settlementAccount; // account address for this order + console.log(allOrders[i]); + const radixWalletAddress = allOrders[i].settlementAccount; // account address for this order + console.log({ radixWalletAddress }); if (!radixWalletAddress) { continue; } - usersDict[radixWalletAddress] = usersDict[radixWalletAddress] - ? usersDict[radixWalletAddress] + 1 - : 1; + usersDict[radixWalletAddress] = usersDict[radixWalletAddress] ? usersDict[radixWalletAddress] + 1 : 1; } + console.log({ usersDict }); return { usersDict: usersDict, totalUsers: Object.keys(usersDict).length, - totalOrders: Object.values(usersDict).reduce((a, b) => a + b), + totalOrders: Object.values(usersDict).reduce((a, b) => a + b, 0), }; }; @@ -84,7 +86,7 @@ const fetchOrdersByPair = async (pairAddress, orderIds) => { } } - return { orders: allOrders }; + return allOrders; }; const getChunkArray = (array, size) => { From 140b89f6bae7c6da87b6509f5d09fb7890a21097 Mon Sep 17 00:00:00 2001 From: dcts Date: Thu, 1 Aug 2024 15:34:34 +0200 Subject: [PATCH 6/9] save result to output file --- scripts/fetchUsers.js | 47 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/scripts/fetchUsers.js b/scripts/fetchUsers.js index b33e0a20..ba17d297 100644 --- a/scripts/fetchUsers.js +++ b/scripts/fetchUsers.js @@ -1,3 +1,6 @@ +const fs = require('fs'); +const path = require('path'); + const fetchUsers = async () => { // fetch all pairs const pairsList = await fetchAllPairs(); @@ -16,15 +19,12 @@ const fetchUsers = async () => { // Process orders to count wallet addresses const usersDict = {}; for (let i = 0; i < allOrders.length; i++) { - console.log(allOrders[i]); const radixWalletAddress = allOrders[i].settlementAccount; // account address for this order - console.log({ radixWalletAddress }); if (!radixWalletAddress) { continue; } usersDict[radixWalletAddress] = usersDict[radixWalletAddress] ? usersDict[radixWalletAddress] + 1 : 1; } - console.log({ usersDict }); return { usersDict: usersDict, @@ -98,9 +98,46 @@ const getChunkArray = (array, size) => { return chunkArray; }; +const getTimestampedFileName = () => { + // Get the current date and time + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, '0'); + const day = String(now.getDate()).padStart(2, '0'); + const hours = String(now.getHours()).padStart(2, '0'); + const minutes = String(now.getMinutes()).padStart(2, '0'); + const seconds = String(now.getSeconds()).padStart(2, '0'); + // Format the filename + return `${year}-${month}-${day}_${hours}${minutes}${seconds}_fetchUsers-output.json`; +} + +const getFilePath = () => { + const filename = getTimestampedFileName(); + const directory = path.join(__dirname, '.scriptOutputs'); + // Ensure the directory exists + if (!fs.existsSync(directory)) { + fs.mkdirSync(directory, { recursive: true }); + } + // Return the full path for the file + return path.join(directory, filename); +} + +const writeObjectToFile = (obj) => { + const jsonString = JSON.stringify(obj, null, 2); + const filePath = getFilePath(); + fs.writeFile(filePath, jsonString, (err) => { + if (err) { + console.error('Error writing file:', err); + } else { + console.log(`Successfully saved output to file: ${filePath}`); + } + }); +} + // RUN SCRIPT (async () => { const result = await fetchUsers(); - console.log(result) -})(); \ No newline at end of file + console.log(result); + writeObjectToFile(result); +})(); From 272c836cc86d1a68244f44b30aa7aa01f045743e Mon Sep 17 00:00:00 2001 From: dcts Date: Thu, 1 Aug 2024 15:34:52 +0200 Subject: [PATCH 7/9] exclude output files from git --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 411baa5a..6a596bec 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,6 @@ next-env.d.ts localhost:3000/ .env + +# Script outputs +*/.scriptOutputs/ \ No newline at end of file From ecfe5bb5611b839ab37aefa0501ce2a4e5e8f277 Mon Sep 17 00:00:00 2001 From: siegfriedbz Date: Fri, 2 Aug 2024 08:55:43 +0800 Subject: [PATCH 8/9] exclude fetchUsers json output from Prettier's formatting checks --- .prettierignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.prettierignore b/.prettierignore index 31e8b092..c3c11673 100644 --- a/.prettierignore +++ b/.prettierignore @@ -3,6 +3,9 @@ # instead of duplicating the .gitignore file, # but duplicating way is more IDE friendly +# fetchUsers json output +scripts/.scriptOutputs/ + # dependencies /node_modules /.pnp From 665d97b6899e376f214fec9770d5d3bcb9b854db Mon Sep 17 00:00:00 2001 From: siegfriedbz Date: Fri, 2 Aug 2024 09:03:25 +0800 Subject: [PATCH 9/9] add eslint-disable-next-line to allow console logs --- scripts/fetchUsers.js | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/scripts/fetchUsers.js b/scripts/fetchUsers.js index ba17d297..7cc3d683 100644 --- a/scripts/fetchUsers.js +++ b/scripts/fetchUsers.js @@ -1,5 +1,5 @@ -const fs = require('fs'); -const path = require('path'); +const fs = require("fs"); +const path = require("path"); const fetchUsers = async () => { // fetch all pairs @@ -23,7 +23,9 @@ const fetchUsers = async () => { if (!radixWalletAddress) { continue; } - usersDict[radixWalletAddress] = usersDict[radixWalletAddress] ? usersDict[radixWalletAddress] + 1 : 1; + usersDict[radixWalletAddress] = usersDict[radixWalletAddress] + ? usersDict[radixWalletAddress] + 1 + : 1; } return { @@ -69,6 +71,7 @@ const fetchOrdersByPair = async (pairAddress, orderIds) => { // fetch all orders by batches of 99 orders for (const chunkOrderIds of chunks) { try { + // eslint-disable-next-line no-console console.log("Fetching batch for " + pairAddress); const response = await fetch("https://api.alphadex.net/v0/pair/orders", { method: "POST", @@ -80,8 +83,8 @@ const fetchOrdersByPair = async (pairAddress, orderIds) => { const data = await response.json(); allOrders.push(...data.orders); - } catch (err) { + // eslint-disable-next-line no-console console.log(err); } } @@ -102,42 +105,43 @@ const getTimestampedFileName = () => { // Get the current date and time const now = new Date(); const year = now.getFullYear(); - const month = String(now.getMonth() + 1).padStart(2, '0'); - const day = String(now.getDate()).padStart(2, '0'); - const hours = String(now.getHours()).padStart(2, '0'); - const minutes = String(now.getMinutes()).padStart(2, '0'); - const seconds = String(now.getSeconds()).padStart(2, '0'); + const month = String(now.getMonth() + 1).padStart(2, "0"); + const day = String(now.getDate()).padStart(2, "0"); + const hours = String(now.getHours()).padStart(2, "0"); + const minutes = String(now.getMinutes()).padStart(2, "0"); + const seconds = String(now.getSeconds()).padStart(2, "0"); // Format the filename return `${year}-${month}-${day}_${hours}${minutes}${seconds}_fetchUsers-output.json`; -} +}; const getFilePath = () => { const filename = getTimestampedFileName(); - const directory = path.join(__dirname, '.scriptOutputs'); + const directory = path.join(__dirname, ".scriptOutputs"); // Ensure the directory exists if (!fs.existsSync(directory)) { fs.mkdirSync(directory, { recursive: true }); } // Return the full path for the file return path.join(directory, filename); -} +}; const writeObjectToFile = (obj) => { const jsonString = JSON.stringify(obj, null, 2); const filePath = getFilePath(); fs.writeFile(filePath, jsonString, (err) => { if (err) { - console.error('Error writing file:', err); + console.error("Error writing file:", err); } else { + // eslint-disable-next-line no-console console.log(`Successfully saved output to file: ${filePath}`); } }); -} - +}; // RUN SCRIPT (async () => { const result = await fetchUsers(); + // eslint-disable-next-line no-console console.log(result); writeObjectToFile(result); })();