From 46adef740e2ef42be5c17a21e4824cc47950d15b Mon Sep 17 00:00:00 2001 From: Swaroop Hegde Date: Mon, 27 Feb 2023 17:52:48 -0800 Subject: [PATCH 01/11] support for offchain consensus as default in dev builds --- pooler/settings/settings.example.json | 2 +- snapshotter_autofill.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pooler/settings/settings.example.json b/pooler/settings/settings.example.json index 282c0bc2..7740d063 100644 --- a/pooler/settings/settings.example.json +++ b/pooler/settings/settings.example.json @@ -118,7 +118,7 @@ "skip_anchor_proof":true }, "consensus":{ - "url": "https://consensus-url", + "url": "http://offchain-consensus:9030", "epoch_tracker_path": "/currentEpoch", "polling_interval":15, "fall_behind_reset_num_blocks":1000, diff --git a/snapshotter_autofill.sh b/snapshotter_autofill.sh index 8c4d6cdd..8e635bd9 100755 --- a/snapshotter_autofill.sh +++ b/snapshotter_autofill.sh @@ -35,7 +35,7 @@ sed -i "s|https://rpc-url|$RPC_URL|" pooler/settings/settings.json sed -i "s|generated-uuid|$UUID|" pooler/settings/settings.json -sed -i "s|https://consensus-url|$consensus_url|" pooler/settings/settings.json +sed -i "s|http://offchain-consensus:9030|$consensus_url|" pooler/settings/settings.json #rm pooler/settings/settings.json.old From 7f93839a37a8094fcd273729c6a5feee464eec47 Mon Sep 17 00:00:00 2001 From: Swaroop Hegde Date: Tue, 28 Feb 2023 17:22:47 -0800 Subject: [PATCH 02/11] reduced pairs to support PowerLoom/deploy#9 --- pooler/settings/settings.example.json | 2 +- pooler/static/cached_pair_addresses_docker.json | 17 ++--------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/pooler/settings/settings.example.json b/pooler/settings/settings.example.json index 7740d063..8ca2500d 100644 --- a/pooler/settings/settings.example.json +++ b/pooler/settings/settings.example.json @@ -67,7 +67,7 @@ "full_nodes": [ { "url": "https://rpc-url", - "rate_limit": "10000000/day;2000/minute;100/second" + "rate_limit": "500000/day;500/minute;10/second" } ], "archive_nodes": [], diff --git a/pooler/static/cached_pair_addresses_docker.json b/pooler/static/cached_pair_addresses_docker.json index db3d0470..810b67bc 100644 --- a/pooler/static/cached_pair_addresses_docker.json +++ b/pooler/static/cached_pair_addresses_docker.json @@ -1,22 +1,9 @@ [ "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc", - "0x21b8065d10f73ee2e260e5b47d3344d3ced7596e", - "0x9928e4046d7c6513326ccea028cd3e7a91c7590a", "0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5", "0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852", - "0xe1573b9d29e2183b1af0e743dc2754979a40d237", - "0xccb63225a7b19dcf66717e4d40c9a72b39331d61", - "0x61b62c5d56ccd158a38367ef2f539668a06356ab", "0x3041cbd36888becc7bbcbc0045e3b1f144466f5f", - "0x9c4fe5ffd9a9fc5678cfbd93aa2d4fd684b67c4c", "0xd3d2e2692501a5c9ca623199d38826e513033a17", - "0x149148acc3b06b8cc73af3a10e84189243a35925", "0xbb2b8038a1640196fbe3e38816f3e67cba72d940", - "0xa478c2975ab1ea89e8196811f51a7b7ade33eb11", - "0x11181bd3baf5ce2a478e98361985d42625de35d1", - "0x7b73644935b8e68019ac6356c40661e1bc315860", - "0x0af81cd5d9c124b4859d65697a4cd10ee223746a", - "0x252b9a9ad1e85d9eff8a3e4782491f87e7109e3a", - "0x470e8de2ebaef52014a47cb5e6af86884947f08c", - "0x582e3da39948c6339433008703211ad2c13eb2ac" - ] \ No newline at end of file + "0xa478c2975ab1ea89e8196811f51a7b7ade33eb11" +] \ No newline at end of file From 4a36a324ea647613abbfcb2c122a8a2fcea2c4ab Mon Sep 17 00:00:00 2001 From: Swaroop Hegde Date: Sun, 5 Mar 2023 21:30:10 -0700 Subject: [PATCH 03/11] add support for pooler auth API PowerLoom/deploy#11 --- Dockerfile | 2 +- pm2.config.js | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8f099454..874c4c2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ RUN chmod +x ./snapshotter_autofill.sh ./init_processes.sh # Expose the port that the application will listen on EXPOSE 8002 -# EXPOSE 9090 +EXPOSE 8555 # Start the application using PM2 # CMD pm2 start pm2.config.js && pm2 logs --lines 100 diff --git a/pm2.config.js b/pm2.config.js index 5b4022d2..feed671b 100644 --- a/pm2.config.js +++ b/pm2.config.js @@ -24,6 +24,15 @@ module.exports = { env: { NODE_ENV: NODE_ENV, } + }, + { + name : "pooler-auth-api", + script : `poetry run python -m pooler.auth.gunicorn_auth_entry_launcher`, + max_restarts: MAX_RESTART, + min_uptime: MIN_UPTIME, + env: { + NODE_ENV: NODE_ENV, + } } ] } From 1c29c96a25499a4217ceba6968ebe88253b99557 Mon Sep 17 00:00:00 2001 From: Swaroop Hegde Date: Mon, 6 Mar 2023 17:48:28 -0700 Subject: [PATCH 04/11] added pair addresses and specific settings for sushiswap and quickswap --- .../settings.quickswapv2-example.json | 141 ++++++++++++++ .../settings.sushiswapv2-example.json | 151 +++++++++++++++ .../cached_pair_addresses_quickswapv2.json | 112 +++++++++++ .../cached_pair_addresses_sushiswapv2.json | 183 ++++++++++++++++++ 4 files changed, 587 insertions(+) create mode 100644 pooler/settings/settings.quickswapv2-example.json create mode 100644 pooler/settings/settings.sushiswapv2-example.json create mode 100644 pooler/static/cached_pair_addresses_quickswapv2.json create mode 100644 pooler/static/cached_pair_addresses_sushiswapv2.json diff --git a/pooler/settings/settings.quickswapv2-example.json b/pooler/settings/settings.quickswapv2-example.json new file mode 100644 index 00000000..6217650e --- /dev/null +++ b/pooler/settings/settings.quickswapv2-example.json @@ -0,0 +1,141 @@ +{ + "namespace": "relevant-namespace", + "core_api": { + "host": "0.0.0.0", + "port": 8002, + "auth": { + "enabled": false, + "header_key": "X-API-KEY" + }, + "public_rate_limit": "10000/day;200/minute;5/second" + }, + "chain_id": 1, + "instance_id": "generated-uuid", + "uniswap_contract_abis":{ + "factory":"pooler/static/abis/IUniswapV2Factory.json", + "router":"pooler/static/abis/UniswapV2Router.json", + "pair_contract":"pooler/static/abis/UniswapV2Pair.json", + "erc20":"pooler/static/abis/IERC20.json", + "trade_events":"pooler/static/abis/UniswapTradeEvents.json" + }, + "logs_prune_time": 3600, + "contract_addresses": { + "iuniswap_v2_factory": "0x5757371414417b8C6CAad45bAeF941aBc7d3Ab32", + "iuniswap_v2_router": "0xa5E0829CaCEd8fFDD4De3c43696c57F7D7A678ff", + "iuniswap_v2_pair": "0xadbF1854e5883eB8aa7BAf50705338739e558E5b", + "USDT": "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", + "DAI": "0x8f3cf7ad23cd3cadbd9735aff958023239c6a063", + "USDC": "0x2791bca1f2de4661ed88a30c99a7a9449aa84174", + "WETH": "0x7ceb23fd6bc0add59e62ac25578270cff1b9f619", + "MAKER": "0x6f7C932e7684666C9fd1d44527765433e01fF61d", + "WETH_USDT": "0xf6422b997c7f54d1c6a6e103bcb1499eea0a7046", + "FRAX": "0x45c32fA6DF82ead1e2EF74d17b76547EDdFaFF89", + "SYN": "0xf8F9efC0db77d8881500bb06FF5D6ABc3070E695", + "FEI": "0x956F47F50A910163D8BF957Cf5846D573E7f87CA", + "agEUR": "0x1a7e4e63778B4f12a199C062f3eFdD288afCBce8", + "DAI_WETH_PAIR": "0x4a35582a710e1f4b2030a3f826da20bfb6703c09", + "USDC_WETH_PAIR": "0x853ee4b2a13f8a742d64c8f088be7ba2131f670d", + "USDT_WETH_PAIR": "0xf6422b997c7f54d1c6a6e103bcb1499eea0a7046" + }, + "uniswap_v2_whitelist": [ + "0x7ceb23fd6bc0add59e62ac25578270cff1b9f619", + "0x2791bca1f2de4661ed88a30c99a7a9449aa84174", + "0x831753dd7087cac61ab5644b308642cc1c33dc13", + "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270", + "0x1bfd67037b42cf73acf2047067bd4f2c47d9bfd6", + "0x8f3cf7ad23cd3cadbd9735aff958023239c6a063", + "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", + "0x9719d867a500ef117cc201206b8ab51e794d3f82", + "0x104592a158490a9228070e0a8e5343b499e125d0", + "0x033d942a6b495c4071083f4cde1f17e986fe856c", + "0xd6df932a45c0f255f85145f286ea0b292b21c90b", + "0xa7051c5a22d963b81d71c2ba64d46a877fbc1821", + "0xfe4546fefe124f30788c4cc1bb9aa6907a7987f9", + "0xE6469Ba6D2fD6130788E0eA9C0a0515900563b59" + ], + "ipfs_url": "/dns/ipfs/tcp/5001", + "uniswap_functions": { + "retrial_attempts": 5 + }, + "rpc": { + "full_nodes": [ + { + "url": "https://rpc-url", + "rate_limit": "500000/day;500/minute;10/second" + } + ], + "archive_nodes": [], + "force_archive_blocks": 100, + "retry": 1, + "request_time_out": 5, + "skip_epoch_threshold_blocks": 30 + }, + "issue_report_url": "http://audit-protocol:9030/reportIssue", + "rlimit": { + "file_descriptors": 2048 + }, + "timeouts": { + "basic": 5, + "archival": 180, + "connection_init": 2 + }, + "module_queues_config":{ + "pair_total_reserves": { + "num_instances": 16 + } + }, + "epoch": { + "height": 10, + "head_offset": 2, + "block_time": 2 + }, + "rabbitmq": { + "user": "guest", + "password": "guest", + "host": "rabbitmq", + "port": 5672, + "setup": { + "core": { + "exchange": "powerloom-backend" + }, + "callbacks": { + "exchange": "powerloom-backend-callbacks", + "path": "pooler/callback_modules/", + "config": "module_queues_config.json" + } + } + }, + + "audit_protocol_engine": { + "url": "http://audit-protocol:9000", + "retry": 5, + "skip_anchor_proof":true + }, + "consensus":{ + "url": "http://offchain-consensus:9030", + "epoch_tracker_path": "/currentEpoch", + "polling_interval":15, + "fall_behind_reset_num_blocks":1000, + "sleep_secs_between_chunks":60 + }, + "redis": { + "host": "redis", + "port": 6379, + "db": 0, + "password": null, + "ssl": false, + "cluster_mode": false + }, + "redis_reader": { + "host": "redis", + "port": 6379, + "db": 0, + "password": null + }, + "webhook_listener": { + "host": "host.docker.internal", + "port": 9090, + "root": "https://pooler-callback.com", + "commit_confirmation_callback_path": "commit_confirm" + } +} diff --git a/pooler/settings/settings.sushiswapv2-example.json b/pooler/settings/settings.sushiswapv2-example.json new file mode 100644 index 00000000..b938a275 --- /dev/null +++ b/pooler/settings/settings.sushiswapv2-example.json @@ -0,0 +1,151 @@ +{ + "namespace": "relevant-namespace", + "core_api": { + "host": "0.0.0.0", + "port": 8002, + "auth": { + "enabled": false, + "header_key": "X-API-KEY" + }, + "public_rate_limit": "10000/day;200/minute;5/second" + }, + "chain_id": 1, + "instance_id": "generated-uuid", + "uniswap_contract_abis":{ + "factory":"pooler/static/abis/ISushiswapFactory.json", + "router":"pooler/static/abis/UniswapV2Router.json", + "pair_contract":"pooler/static/abis/SushiswapPair.json", + "erc20":"pooler/static/abis/IERC20.json", + "trade_events":"pooler/static/abis/UniswapTradeEvents.json" + }, + "logs_prune_time": 3600, + "contract_addresses": { + "iuniswap_v2_factory": "0xC0AEe478e3658e2610c5F7A4A2E1777cE9e4f2Ac", + "iuniswap_v2_router": "0xd9e1cE17f2641f24aE83637ab66a2cca9C378B9F", + "USDT": "0xdac17f958d2ee523a2206206994597c13d831ec7", + "DAI": "0x6b175474e89094c44da98b954eedeac495271d0f", + "USDC": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", + "WETH": "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", + "MAKER": "0x9f8F72aA9304c8B593d555F12eF6589cC3A579A2", + "WETH_USDT": "0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852", + "FRAX": "0x853d955aCEf822Db058eb8505911ED77F175b99e", + "SYN": "0x0f2D719407FdBeFF09D87557AbB7232601FD9F29", + "FEI": "0x956F47F50A910163D8BF957Cf5846D573E7f87CA", + "agEUR": "0x1a7e4e63778B4f12a199C062f3eFdD288afCBce8", + "DAI_WETH_PAIR": "0xc3d03e4f041fd4cd388c549ee2a29a9e5075882f", + "USDC_WETH_PAIR": "0x397ff1542f962076d0bfe58ea045ffa2d347aca0", + "USDT_WETH_PAIR": "0x06da0fd433c1a5d7a4faa01111c044910a184553" + }, + "uniswap_v2_whitelist": [ + "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", + "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599", + "0x6b175474e89094c44da98b954eedeac495271d0f", + "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", + "0x383518188c0c6d7730d91b2c03a03c837814a899", + "0xdac17f958d2ee523a2206206994597c13d831ec7", + "0x0000000000085d4780b73119b644ae5ecd22b376", + "0x5d3a536e4d6dbd6114cc1ead35777bab948e3643", + "0x57ab1ec28d129707052df4df418d58a2d46d5f51", + "0x514910771af9ca656af840dff83e8264ecf986ca", + "0x0bc529c00c6401aef6d220be8c6ea1667f6ad93e", + "0x8798249c2e607446efb7ad49ec89dd1865ff4272", + "0x1456688345527be1f37e9e627da0837d6f08c925", + "0x3449fc1cd036255ba1eb19d65ff4ba2b8903a69a", + "0x2ba592f78db6436527729929aaf6c908497cb200", + "0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0", + "0xa1faa113cbe53436df28ff0aee54275c13b40975", + "0xdb0f18081b505a7de20b18ac41856bcb4ba86a1a", + "0x04fa0d235c4abf4bcf4787af4cf447de572ef828", + "0x3155ba85d5f96b2d030a4966af206230e46849cb", + "0x87d73e916d7057945c9bcd8cdd94e42a6f47f776", + "0xdfe66b14d37c77f4e9b180ceb433d1b164f0281d", + "0xad32a8e6220741182940c5abf610bde99e737b2d", + "0xafcE9B78D409bF74980CACF610AFB851BF02F257", + "0x6b3595068778dd592e39a122f4f5a5cf09c90fe2" + ], + "ipfs_url": "/dns/ipfs/tcp/5001", + "uniswap_functions": { + "retrial_attempts": 5 + }, + "rpc": { + "full_nodes": [ + { + "url": "https://rpc-url", + "rate_limit": "500000/day;500/minute;10/second" + } + ], + "archive_nodes": [], + "force_archive_blocks": 100, + "retry": 1, + "request_time_out": 5, + "skip_epoch_threshold_blocks": 30 + }, + "issue_report_url": "http://audit-protocol:9030/reportIssue", + "rlimit": { + "file_descriptors": 2048 + }, + "timeouts": { + "basic": 5, + "archival": 180, + "connection_init": 2 + }, + "module_queues_config":{ + "pair_total_reserves": { + "num_instances": 16 + } + }, + "epoch": { + "height": 10, + "head_offset": 2, + "block_time": 2 + }, + "rabbitmq": { + "user": "guest", + "password": "guest", + "host": "rabbitmq", + "port": 5672, + "setup": { + "core": { + "exchange": "powerloom-backend" + }, + "callbacks": { + "exchange": "powerloom-backend-callbacks", + "path": "pooler/callback_modules/", + "config": "module_queues_config.json" + } + } + }, + + "audit_protocol_engine": { + "url": "http://audit-protocol:9000", + "retry": 5, + "skip_anchor_proof":true + }, + "consensus":{ + "url": "http://offchain-consensus:9030", + "epoch_tracker_path": "/currentEpoch", + "polling_interval":15, + "fall_behind_reset_num_blocks":1000, + "sleep_secs_between_chunks":60 + }, + "redis": { + "host": "redis", + "port": 6379, + "db": 0, + "password": null, + "ssl": false, + "cluster_mode": false + }, + "redis_reader": { + "host": "redis", + "port": 6379, + "db": 0, + "password": null + }, + "webhook_listener": { + "host": "host.docker.internal", + "port": 9090, + "root": "https://pooler-callback.com", + "commit_confirmation_callback_path": "commit_confirm" + } +} diff --git a/pooler/static/cached_pair_addresses_quickswapv2.json b/pooler/static/cached_pair_addresses_quickswapv2.json new file mode 100644 index 00000000..98bbb003 --- /dev/null +++ b/pooler/static/cached_pair_addresses_quickswapv2.json @@ -0,0 +1,112 @@ +[ + "0x369582d2010b6ed950b571f4101e3bb9b554876f", + "0x853ee4b2a13f8a742d64c8f088be7ba2131f670d", + "0x160532d2536175d65c03b97b0630a9802c274dad", + "0xadbf1854e5883eb8aa7baf50705338739e558e5b", + "0x6e7a5fafcec6bb1e78bae2a1f0b612012bf14827", + "0x65752c54d9102bdfd69d351e1838a1be83c924c6", + "0x2cf7252e74036d1da831d11089d326296e64a728", + "0xdc9232e2df177d7a12fdff6ecbab114e2231198d", + "0x6e53cb6942e518376e9e763554db1a45ddcd25c4", + "0xf6422b997c7f54d1c6a6e103bcb1499eea0a7046", + "0x4a35582a710e1f4b2030a3f826da20bfb6703c09", + "0x604229c960e5cacf2aaeac8be68ac07ba9df81c3", + "0xf04adbf75cdfc5ed26eea4bbbb991db002036bdd", + "0x096c5ccb33cfc5732bcd1f3195c13dbefc4c82f4", + "0xb0e69f24982791dd49e316313fd3a791020b8bf7", + "0xf6a637525402643b0654a54bead2cb9a83c8b498", + "0x9a8b2601760814019b7e6ee0052e25f1c623d1e6", + "0x3902b89e0e28a23f82086b87bb81ef5a1eabf276", + "0x7c303894a165830751f524ebdb6b198affbb7211", + "0x304e57c752e854e9a233ae82fcc42f7568b81180", + "0x9e3880647c07ba13e65663de29783ecd96ec21de", + "0x17a7829cc1167ecda8b9668414a5405050846f8a", + "0xaab5254e17380511887aaba7e96a5339a519e26a", + "0x0213695eb4cf4f795397fcc6d3cd370143838e36", + "0x7dab6054518f67ac69b0afa4c46ff16dd2bf560e", + "0x7a602815908e1615393148a7880a7fc9e57949ae", + "0xca7300804f0fbcec8208bf2036ca0c78059f7a0a", + "0x56ea2342232ce09bc204e5f7d3641df4303bbd8a", + "0xf3eb2f17eafbf35e92c965a954c6e7693187057d", + "0x60e70705b52a4a5bdc1d8614426ba5016a68ab38", + "0x74214f5d8aa71b8dc921d8a963a1ba3605050781", + "0x71bd159cf9136d038a60f10118741dfcb10c3111", + "0x55a08b3c363a2cc61a07d54b52fee201bd4ec0dc", + "0xdf298db9aa32641123c395b7c44d1d5b94ffdb64", + "0x019ba0325f1988213d448b3472fa1cf8d07618d7", + "0xfc2fc983a411c4b1e238f7eb949308cf0218c750", + "0x69437901da6952da49bca49ffc923dfb82abfb16", + "0xa5cabfc725dfa129f618d527e93702d10412f039", + "0xf41a758bede6c872942e7e85c7f0ed478658641d", + "0xf3c62dbbfec92a2e73d676d62ebec06a6bc224e2", + "0x34be3b86f8565fa130a4f8a9349aef3f2c9aa5a7", + "0xccb9d2100037f1253e6c1682adf7dc9944498aff", + "0xa022ae9cfadefd62d70b510c27dc3d5db67ca43b", + "0xe89fae1b4ada2c869f05a0c96c87022dadc7709a", + "0x8f89216177144a553be966fd45e17f9e915e3ef4", + "0x90bc3e68ba8393a3bf2d79309365089975341a43", + "0xeabf86e10a7d0475000147cfd5ac20b98b0347f9", + "0x9fb51819ccf45dcc6610a876f27c692272bc65fa", + "0xc34f686947df1e91e9709777cb70bc8a5584ce92", + "0x01ebd3e57f4af47b7e96240e2b7b2227c902614a", + "0xb2b6d423e535b57aad06e9866803b95fb66152ea", + "0xe88e24f49338f974b528ace10350ac4576c5c8a1", + "0xde4fea426e045374a2f629a28561978dc86677c5", + "0x5a94f81d25c73eddbdd84b84e8f6d36c58270510", + "0xf64d2b41ca5392ec86d519d616603d2bb85b2a5d", + "0x9c3e6de086632973bce6e5814f7c621cd1799a3d", + "0x1f1e4c845183ef6d50e9609f16f6f9cae43bc9cb", + "0x5ca6ca6c3709e1e6cfe74a50cf6b2b6ba2dadd67", + "0xa5bf14bb945297447fe96f6cd1b31b40d31175cb", + "0x8fcb93c0d9fa361c57c1d31bbf1c4d8c61af7d3d", + "0x05efa0ed56dddb4e950e3f5a54e349a137d4edc9", + "0x68c2f8aa7d07948b2c36f58269189f0d66497fa4", + "0x690a87ed8972e451e755b8f2dc1fc2b28e3c6566", + "0x032d86f50ee9c0c361f268bd7c08013465e29968", + "0xbfb1580eb61c9a7a00eebfb8aec867a65e100a3e", + "0xff4a55f5fc202f27e589a0e915a1d151ed4fe6dc", + "0x9b5c71936670e9f1f36e63f03384de7e06e60d2a", + "0x57f34a01fadb760a218343e36299ffa29cc9da62", + "0x74bf9bef07ff54f5c70786fa4bdad45a3cca5625", + "0x747375305b825c49fb97ee0ac09d19ec9ef94bd2", + "0xa0f330f5fc47ee7a3297dbd6be6fe60cd0346b26", + "0x580da9c2b04e30d79ffd130698909ef2517cdd0f", + "0x1bd06b96dd42ada85fdd0795f3b4a79db914add5", + "0xf1464fabaffff79eea51be4e9c883c1bc5737ab2", + "0xc41b826985e4fe7cbb314e323b74e26eeacf0535", + "0x4deb2d750b790f2a1a81ed51d9694675ae8628b8", + "0x85ba262be13329a2db5acf9aa46ac2345b5df4ff", + "0x320a50f32fb9e20fe113573031132c89835e496c", + "0xdc37ddbf79d98b3a7c3c11426e33fe440b8dd806", + "0x8b1fd78ad67c7da09b682c5392b65ca7caa101b9", + "0x7ee7d075d2a74a2a05e8837a6a4727fab5d23c5b", + "0x83e5f110918668ef9518ee4263619998778af31b", + "0xd8770553b6daebeb77b29959d9589638e812b5cc", + "0x2d252d4a903a450afa9dac54cb696f0690259a62", + "0x88b48ef8cdab9f48db44bd2e7af7b725a41bd6a9", + "0xaddc9c73f3cbad4e647eaff691715898825ac20c", + "0x8b4e00810c927bb1c02dee73d714a31121689ab3", + "0xe44d0a5b14e763f77b4aadcc12be20e153a8e5fc", + "0x2a574629ca405fa43a8f21faa64ff73dd320f45b", + "0x78fd833ec4464d3d6d470e5853dbfc2de0ca6f5b", + "0x6f9e92dd4734c168a734b873dc3db77e39552eb6", + "0xf9845837bec7c7310560cb0f1b84860c1ab6a7ca", + "0x0a8a3cb9a21c893a207826e76125ef6faaad99ec", + "0xde6daa8d1580545f07264dda9cfc40c3d6862b1a", + "0xc4bf2a012af69d44abc4bbe2b1875a222c1c32e1", + "0x7e2eaa9514c7a56b4d556ff0d1009cb9c7f2b89d", + "0xeef611894ceae652979c9d0dae1deb597790c6ee", + "0x07d53b147ef96fad1896d1156755a9da7e06098e", + "0x59377dbd9f550568540cae0b21412bee1cc3f439", + "0xff2bbcb399ad50bbd06debadd47d290933ae1038", + "0x83b0bd12077b017d22a18fa4d09a560bd8fd97b3", + "0xfca921d46ab12b54eb22aabff798a4c861419ae0", + "0xc089c10fb638f36657873137828618ab02bd34aa", + "0xb5a7c572741d77f34d2096f928beb6168f31a621", + "0x290a4793b4e972188482b8046e7ea2ea5d3fdbaf", + "0x10e14e23686826915efcb23ed9b128750831bf4c", + "0x8bab87ecf28bf45507bd745bc70532e968b5c2de", + "0x82a54e66c05fcd555adae593848a4257c9e51ad9", + "0x72cf5ee9ee918a529b25bbcb0372594008178535", + "0x59153f27eefe07e5ece4f9304ebba1da6f53ca88" +] \ No newline at end of file diff --git a/pooler/static/cached_pair_addresses_sushiswapv2.json b/pooler/static/cached_pair_addresses_sushiswapv2.json new file mode 100644 index 00000000..30768f46 --- /dev/null +++ b/pooler/static/cached_pair_addresses_sushiswapv2.json @@ -0,0 +1,183 @@ +[ + "0x6a091a3406e0073c3cd6340122143009adac0eda", + "0x397ff1542f962076d0bfe58ea045ffa2d347aca0", + "0x055475920a8c93cffb64d039a8205f7acc7722d3", + "0x06da0fd433c1a5d7a4faa01111c044910a184553", + "0xceff51756c56ceffca006cd410b03ffc46dd3a58", + "0x69b81152c5a8d35a67b32a4d3772795d96cae4da", + "0x795065dcc9f64b5614c407a6efdc400da6221fb0", + "0xd4e7a6e2d03e4e48dfc27dd3f46df1c176647e38", + "0xe12af1218b4e9272e9628d7c7dc6354d137d024e", + "0xd75ea151a61d06868e31f8988d28dfe5e9df57b4", + "0xc3f279090a47e80990fe3a9c30d24cb117ef91a8", + "0xc3d03e4f041fd4cd388c549ee2a29a9e5075882f", + "0x0463a06fbc8bf28b3f120cd1bfc59483f099d332", + "0x4a86c01d67965f8cb3d0aaa2c655705e64097c31", + "0xc558f600b34a5f69dd2f0d06cb8a88d829b7420a", + "0xa7a8edfda2b8bf1e5084e2765811effee21ef918", + "0xc44676d5bce2dc078e50008f07c31e4aaebbf110", + "0x1498bd576454159bb81b5ce532692a8752d163e8", + "0xdb06a76733528761eda47d356647297bc35a98bd", + "0x088ee5007c98a9677165d78dd2109ae4a3d04d0c", + "0x117d4288b3635021a3d612fe05a3cbf5c717fef2", + "0x61eb53ee427ab4e007d78a9134aacb3101a2dc23", + "0x99b42f2b49c395d2a77d973f6009abb5d67da343", + "0x31503dcb60119a812fee820bb7042752019f2355", + "0x82917fb0dd65b0e5c85eea66e4f5c1ed484bc629", + "0x36e2fcccc59e5747ff63a03ea2e5c0c2c14911e7", + "0x495f8ef80e13e9e1e77d60d2f384bb49694823ef", + "0x05104ebba2b6d3b8254aa41cf6df80462f6160ae", + "0x1e888882d0f291dd88c5605108c72d414f29d460", + "0x93e2f3a8277e0360081547d711446e4a1f83546d", + "0xc40d16476380e4037e6b1a2594caf6a6cc8da967", + "0x281df7fc89294c84afa2a21ffee8f6807f9c9226", + "0xe9cf42bfcd65e342a96fc063304e22db33e236ac", + "0x18cd890f4e23422dc4aa8c2d6e0bd3f3bd8873d8", + "0x58dc5a51fe44589beb22e8ce67720b5bc5378009", + "0xfe9a776154f30281517d4f06578591911906f4e1", + "0xb84c45174bfc6b8f3eaecbae11dee63114f5c1b2", + "0x611cde65dea90918c0078ac0400a72b0d25b9bb1", + "0x82ecdd4635766560c4e3a8efa8a7c1fff566111e", + "0xc96f20099d96b37d7ede66ff9e4de59b9b1065b1", + "0x82dbc2673e9640343d263a3c55de49021ad39ae2", + "0xb5de0c3753b6e1b4dba616db82767f17513e6d4e", + "0xc08ed9a9abeabcc53875787573dc32eee5e43513", + "0x1bec4db6c3bc499f3dbf289f5499c30d541fec97", + "0xa1181481beb2dc5de0daf2c85392d81c704bf75d", + "0x18d98d452072ac2eb7b74ce3db723374360539f1", + "0xfceaaf9792139bf714a694f868a215493461446d", + "0x559ebe4e206e6b4d50e9bd3008cda7ce640c52cb", + "0xdbb69ea87507525fffbd1c4f1ad6f7d30a9a402e", + "0x9461173740d27311b176476fa27e94c681b1ea6b", + "0x110492b31c59716ac47337e616804e3e3adc0b4a", + "0x0e26a21013f2f8c0362cfae608b4e69a249d5efc", + "0xaf988aff99d3d0cb870812c325c588d8d8cb7de8", + "0x05767d9ef41dc40689678ffca0608878fb3de906", + "0xdd51121d1efc398b4c09fd0cb84d79ae2c923fc9", + "0xf1e34d19f63b69eaa70952f2f64f735849959833", + "0xa5c475167f03b1556c054e0da78192cd2779087f", + "0x3d3f13f2529ec3c84b2940155effbf9b39a8f3ec", + "0x753f33c13fe44d41a8cc6ac202a6de6c53c58b6a", + "0x9cd028b1287803250b1e226f0180eb725428d069", + "0x1b803af96f37ae51fa753e13e1d2e372dfa17791", + "0x9a0cc6791a5409ce3547f1f1d00e058c79d0a72c", + "0x1f4c763bde1d4832b3ea0640e66da00b98831355", + "0xe9ab8038ee6dd4fcc7612997fe28d4e22019c4b4", + "0xbb19141e045b133169d7c7160c5e54a54cc821b2", + "0x37922c69b08babcceae735a31235c81f1d1e8e43", + "0x0bec54c89a7d9f15c4e7faa8d47adedf374462ed", + "0xe93b1b5e1dadce8152a69470c1b31463af260296", + "0xbbbdb106a806173d1eea1640961533ff3114d69a", + "0x161a4987adc70a642a247bd689a54efe3190be42", + "0x31fa985bb0c282a814e7f3f0dce88b2a44197f60", + "0x72ea6ca0d47b337f1ea44314d9d90e2a897edaf5", + "0x328dfd0139e26cb0fef7b0742b49b0fe4325f821", + "0x15a8e38942f9e353bec8812763fb3c104c89ecf4", + "0xdafd66636e2561b0284edde37e42d192f2844d40", + "0x60aacb5e507d41a95c9109cc6778fb0b94227616", + "0xa1d7b2d891e3a1f9ef4bbc5be20630c2feb1c470", + "0xa87e2c5d5964955242989b954474ff2eb08dd2f5", + "0xf7ac1f571856d7dbfbb56728c48823a2a19e4326", + "0x7825de5586e4d2fd04459091bbe783fa243e1bf3", + "0x9a13867048e01c663ce8ce2fe0cdae69ff9f35e3", + "0xce7e98d4da6ebda6af474ea618c6b175729cd366", + "0xf13eef1c6485348b9c9fa0d5df2d89accc5b0147", + "0x6469b34a2a4723163c4902dbbdea728d20693c12", + "0x698abbbc986c59d02941e18bc96fe2396493339b", + "0x0a2f9b5360b5c7b6d3ce826971425b3b8b766519", + "0x94b86ca6f7a495930fe7f552eb9e4cbb5ef2b736", + "0xbe71372995e8e920e4e72a29a51463677a302e8d", + "0xc5817a4c5e8ec6488c9a26c6862ff3060757b498", + "0x53162d78dca413d9e28cf62799d17a9e278b60e8", + "0x2980bfad83d889617cc9a019d1c9a18dd5722504", + "0xe339c1d0a744053cbceb0d2dc2d13967c8a69586", + "0xea9b00b169dda4cbc9fa0a64b0d1e4a6a23a3f34", + "0x9d45081706102e7aaddd0973268457527722e274", + "0x95cfa1f48fad82232772d3b1415ad4393517f3b5", + "0x0f8086d08a69ebd8e3a130a87a3b6a260723976f", + "0x0e0b5c0c5a38cef295dbb1366caa683564e31d40", + "0xeefa3b448768dd561af4f743c9e925987a1f8d09", + "0x7cf6a842e72c4bd7bc92e0c17ebdaa95ac1e7825", + "0xa914a9b9e03b6af84f9c6bd2e0e8d27d405695db", + "0x0d8a21f2ea15269b7470c347083ee1f85e6a723b", + "0x0f82e57804d0b1f6fab2370a43dcfad3c7cb239c", + "0xbcd6a2ddafbaa7f424698ed69e717c0c0f1e99bf", + "0x5c596c6a65f628fc1090853d8eb1927651e9d9b2", + "0x164fe0239d703379bddde3c80e4d4800a1cd452b", + "0xfca032a9a68c5cbfaf8377775173eca59b837073", + "0x0eee7f7319013df1f24f5eaf83004fcf9cf49245", + "0x17890deb188f2de6c3e966e053da1c9a111ed4a5", + "0x07d5695a24904cc1b6e3bd57cc7780b90618e3c4", + "0x29c827ce49accf68a1a278c67c9d30c52fbbc348", + "0xa75f7c2f025f470355515482bde9efa8153536a8", + "0xe55c3e83852429334a986b265d03b879a3d188ac", + "0x5ba61c0a8c4dcccc200cd0ccc40a5725a426d002", + "0xf55c33d94150d93c2cfb833bcca30be388b14964", + "0x10b47177e92ef9d5c6059055d92ddf6290848991", + "0x208226200b45b82212b814f49efa643980a7bdd1", + "0x295685c8fe08d8192981d21ea1fe856a07443920", + "0xf169cea51eb51774cf107c88309717dda20be167", + "0x70b8ddb05bdf5400daef4d6dbb0f14f366c00676", + "0x9c051f8a6648a51ef324d30c235da74d060153ac", + "0xe06f8d30ac334c857fc8c380c85969c150f38a6a", + "0xba13afecda9beb75de5c56bbaf696b880a5a50dd", + "0x04e02423299c2c56c451ceb125779cfa0f7ac6bc", + "0xec87cb93ecd0fe2f80dbd0d85431eacf1de09d50", + "0x2a93167ed63a31f35ca4788e2eb9fbd9fa6089d0", + "0x64c2f792038f1fb55da1a9a22749971eac94463e", + "0xe4ebd836832f1a8a81641111a5b081a2f90b9430", + "0x817211d408fbc19193e482f4900354e4cc894a44", + "0x7c959aa82761c2eeb9b8cba504f90df0a7489a53", + "0xb301d7efb4d46528f9cf0e5c86b065fbc9f50e9a", + "0x8f9ef75cd6e610dd8acf8611c344573032fb9c3d", + "0x70813e6d7550fa1a31ef6028eb60c4d72285c7f0", + "0x45b6ffb13e5206dafe2cc8780e4ddc0e32496265", + "0xf79321e80acd5fa590936f09acb90ec6471fcbc4", + "0xccb42aaab2ea1f8012c30418ee3fce550954d827", + "0x83b546e10917432a722444672504f0d459472171", + "0x001b6450083e531a5a7bf310bd2c1af4247e23d4", + "0x438a6b4026017a4be5c16f7be45e7c85e5cb86ac", + "0x87b918e76c92818db0c76a4e174447aee6e6d23f", + "0x75382c52b6f90b3f8014bfcadac2386513f1e3bc", + "0xb50580b0d81d9fe860746387cef9a8fc36d48d49", + "0x481ddaf90c59d91f3e480e6793122e62612ca5a9", + "0x3349217670f9aa55c5640a2b3d806654d27d0569", + "0xc2ce29af8930fc32ca24ccb76c2866fb2bf692fe", + "0x41e05211a0de162b4a131730a500f114f653a0ad", + "0x97f34c8e5992eb985c5f740e7ee8c7e48a1de76a", + "0x0225e940deecc32a8d7c003cfb7dae22af18460c", + "0x5fa4370164a2fabeef159b893299d59ff5dc1e6d", + "0x5a70513ec742ebbfb99094c7fd3096a0a3e25b86", + "0xf1f85b2c54a2bd284b1cf4141d64fd171bd85539", + "0x26e6a391953169d141df36834acfd9c61d3d4d59", + "0x77337ff10206480739a768124a18f3aa8c089153", + "0x23a9292830fc80db7f563edb28d2fe6fb47f8624", + "0x938625591adb4e865b882377e2c965f9f9b85e34", + "0xef4fa701fa5df88ef0d5707d6144e446a1a2238b", + "0x22b15c7ee1186a7c7cffb2d942e20fc228f6e4ed", + "0xb124c4e18a282143d362a066736fd60d22393ef4", + "0x877d9c970b8b5501e95967fe845b7293f63e72f7", + "0x7bc735d6974f6153b1de24e40c0bf5715ca7fe1d", + "0x91a48c69ec3f3ce855fe5054f82d2bef8fd66c43", + "0xcb2286d9471cc185281c4f763d34a962ed212962", + "0x0c6f06b32e6ae0c110861b8607e67da594781961", + "0x5b64c8d934b5e180cf1ca814584c2655d81c5b5c", + "0x34b13f8cd184f55d0bd4dd1fe6c07d46f245c7ed", + "0xeaadde1e14c587a7fb4ba78ea78109bb32975f1e", + "0xc7175038323562cb68e4bbdd379e9fe65134937f", + "0x4708713b4b6bd32e41bcb2f9c5901d74fedba447", + "0xbe1e98685fb293144325440c16f69954ffcb790c", + "0x8b11c6a6e4bb220641561184396d76ee6103bf26", + "0x0589e281d35ee1acf6d4fd32f1fba60effb5281b", + "0xc83ce8612164ef7a13d17ddea4271dd8e8eebe5d", + "0xbfbe3fdb93adb2d5899ac34cd189bebc86e08f5d", + "0x557b08a0cab46bfe22471b65522757ea92a9e7a5", + "0x18a797c7c70c1bf22fdee1c09062aba709cacf04", + "0x42b7b8f8f83fa5cbf0176f8c24ad51ebcd4b5f17", + "0xd829de54877e0b66a2c3890b702fa5df2245203e", + "0x195b2ed7dfb2bb19c63b8b06677d46934c0c4eea", + "0x1c934bc86839b5718a1bdf69880de56f65388531", + "0x55d8ec728ea72477c6db12ca497a803c8db361e9", + "0x79e0d4858af8071349469b6589a3c23c1fe1586e", + "0x75507bab932fc47531bf1654f243c088a3fa1f69" +] \ No newline at end of file From 178052fe53b21972a8769602b670a5af89725f58 Mon Sep 17 00:00:00 2001 From: Swaroop Hegde Date: Thu, 9 Mar 2023 01:53:49 -0800 Subject: [PATCH 05/11] fixed chain id --- pooler/settings/settings.quickswapv2-example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pooler/settings/settings.quickswapv2-example.json b/pooler/settings/settings.quickswapv2-example.json index 6217650e..818bd129 100644 --- a/pooler/settings/settings.quickswapv2-example.json +++ b/pooler/settings/settings.quickswapv2-example.json @@ -9,7 +9,7 @@ }, "public_rate_limit": "10000/day;200/minute;5/second" }, - "chain_id": 1, + "chain_id": 137, "instance_id": "generated-uuid", "uniswap_contract_abis":{ "factory":"pooler/static/abis/IUniswapV2Factory.json", From ed519566b5e950f89b3e1a15f4f016366357e447 Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Mon, 29 May 2023 15:50:49 +0530 Subject: [PATCH 06/11] chore: Update example files and autofill scripts --- .gitignore | 1 + config/aggregator.example.json | 78 ++++++++++++++++++++++--------- config/projects.example.json | 30 +----------- config/settings.example.json | 22 ++++----- snapshotter_autofill.sh | 85 +++++++++++++++++++++++++++------- 5 files changed, 139 insertions(+), 77 deletions(-) diff --git a/.gitignore b/.gitignore index 2509b5b8..96e716e9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ uniswapTokenData **/*projects.json **/*aggregator.json **/*settings.json +**/*.backup diff --git a/config/aggregator.example.json b/config/aggregator.example.json index 33bba704..c8eb8e1f 100644 --- a/config/aggregator.example.json +++ b/config/aggregator.example.json @@ -1,6 +1,5 @@ { - "config": [ - { + "config": [{ "project_type": "aggregate_uniswap_pairContract_24h_trade_volume", "aggregate_on": "SingleProject", "filters": { @@ -23,13 +22,23 @@ } }, { - "project_type": "aggregate_uniswap_24h_top_pairs", + "project_type": "aggregate_uniswap_24h_top_pairs_lite", "aggregate_on": "MultiProject", "projects_to_wait_for": [ + "aggregate_uniswap_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod", "aggregate_uniswap_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0x11181bd3baf5ce2a478e98361985d42625de35d1:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0x11181bd3baf5ce2a478e98361985d42625de35d1:UNISWAPV2-ph15-prod" + "uniswap_pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod" ], "processor": { "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_pairs_24h", @@ -37,39 +46,64 @@ } }, { - "project_type": "aggregate_uniswap_7d_top_pairs", + "project_type": "aggregate_uniswap_24h_top_tokens_lite", "aggregate_on": "MultiProject", "projects_to_wait_for": [ - "aggregate_uniswap_pairContract_7d_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_7d_trade_volume:0x11181bd3baf5ce2a478e98361985d42625de35d1:UNISWAPV2-ph15-prod" + "aggregate_uniswap_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod" ], "processor": { - "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_pairs_7d", - "class_name": "AggreagateTopPairsProcessor" + "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_tokens", + "class_name": "AggreagateTopTokensProcessor" } }, { - "project_type": "aggregate_uniswap_24h_top_tokens", + "project_type": "aggregate_uniswap_7d_top_pairs_lite", "aggregate_on": "MultiProject", "projects_to_wait_for": [ - "aggregate_uniswap_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0x11181bd3baf5ce2a478e98361985d42625de35d1:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0x11181bd3baf5ce2a478e98361985d42625de35d1:UNISWAPV2-ph15-prod" + "aggregate_uniswap_pairContract_7d_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_7d_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_7d_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_7d_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_7d_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_7d_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_7d_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod" ], "processor": { - "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_tokens", - "class_name": "AggreagateTopTokensProcessor" + "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_top_pairs_7d", + "class_name": "AggreagateTopPairsProcessor" } }, { - "project_type": "aggregate_uniswap_24h_stats", + "project_type": "aggregate_uniswap_24h_stats_lite", "aggregate_on": "MultiProject", "projects_to_wait_for": [ + "aggregate_uniswap_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2-ph15-prod", + "aggregate_uniswap_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod", + "uniswap_pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2-ph15-prod", "aggregate_uniswap_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod", - "aggregate_uniswap_pairContract_24h_trade_volume:0x11181bd3baf5ce2a478e98361985d42625de35d1:UNISWAPV2-ph15-prod", - "uniswap_pairContract_pair_total_reserves:0x11181bd3baf5ce2a478e98361985d42625de35d1:UNISWAPV2-ph15-prod" + "uniswap_pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2-ph15-prod" ], "processor": { "module": "pooler.modules.uniswapv2.aggregate.multi_uniswap_stats", diff --git a/config/projects.example.json b/config/projects.example.json index 63be3e24..8c8576ef 100644 --- a/config/projects.example.json +++ b/config/projects.example.json @@ -3,25 +3,12 @@ "project_type": "uniswap_pairContract_pair_total_reserves", "projects":[ "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc", - "0x21b8065d10f73ee2e260e5b47d3344d3ced7596e", - "0x9928e4046d7c6513326ccea028cd3e7a91c7590a", "0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5", "0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852", - "0xe1573b9d29e2183b1af0e743dc2754979a40d237", - "0xccb63225a7b19dcf66717e4d40c9a72b39331d61", - "0x61b62c5d56ccd158a38367ef2f539668a06356ab", "0x3041cbd36888becc7bbcbc0045e3b1f144466f5f", - "0x9c4fe5ffd9a9fc5678cfbd93aa2d4fd684b67c4c", "0xd3d2e2692501a5c9ca623199d38826e513033a17", - "0x149148acc3b06b8cc73af3a10e84189243a35925", "0xbb2b8038a1640196fbe3e38816f3e67cba72d940", - "0xa478c2975ab1ea89e8196811f51a7b7ade33eb11", - "0x11181bd3baf5ce2a478e98361985d42625de35d1", - "0x7b73644935b8e68019ac6356c40661e1bc315860", - "0x0af81cd5d9c124b4859d65697a4cd10ee223746a", - "0x252b9a9ad1e85d9eff8a3e4782491f87e7109e3a", - "0x470e8de2ebaef52014a47cb5e6af86884947f08c", - "0x582e3da39948c6339433008703211ad2c13eb2ac" + "0xa478c2975ab1ea89e8196811f51a7b7ade33eb11" ], "processor":{ "module": "pooler.modules.uniswapv2.pair_total_reserves", @@ -32,25 +19,12 @@ "project_type": "uniswap_pairContract_trade_volume", "projects":[ "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc", - "0x21b8065d10f73ee2e260e5b47d3344d3ced7596e", - "0x9928e4046d7c6513326ccea028cd3e7a91c7590a", "0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5", "0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852", - "0xe1573b9d29e2183b1af0e743dc2754979a40d237", - "0xccb63225a7b19dcf66717e4d40c9a72b39331d61", - "0x61b62c5d56ccd158a38367ef2f539668a06356ab", "0x3041cbd36888becc7bbcbc0045e3b1f144466f5f", - "0x9c4fe5ffd9a9fc5678cfbd93aa2d4fd684b67c4c", "0xd3d2e2692501a5c9ca623199d38826e513033a17", - "0x149148acc3b06b8cc73af3a10e84189243a35925", "0xbb2b8038a1640196fbe3e38816f3e67cba72d940", - "0xa478c2975ab1ea89e8196811f51a7b7ade33eb11", - "0x11181bd3baf5ce2a478e98361985d42625de35d1", - "0x7b73644935b8e68019ac6356c40661e1bc315860", - "0x0af81cd5d9c124b4859d65697a4cd10ee223746a", - "0x252b9a9ad1e85d9eff8a3e4782491f87e7109e3a", - "0x470e8de2ebaef52014a47cb5e6af86884947f08c", - "0x582e3da39948c6339433008703211ad2c13eb2ac" + "0xa478c2975ab1ea89e8196811f51a7b7ade33eb11" ], "processor":{ "module": "pooler.modules.uniswapv2.trade_volume", diff --git a/config/settings.example.json b/config/settings.example.json index faedb44e..cfb12968 100644 --- a/config/settings.example.json +++ b/config/settings.example.json @@ -51,8 +51,8 @@ } }, "reporting": { - "slack_url": "", - "service_url": "http://reporting-service:9030/reportIssue" + "slack_url": "https://slack-reporting-url", + "service_url": "https://powerloom-reporting-url" }, "redis": { "host": "redis", @@ -77,22 +77,22 @@ "pair_contract_abi": "pooler/static/abis/UniswapV2Pair.json", "protocol_state": { "abi": "pooler/static/abis/ProtocolContract.json", - "address": "0x568355e16e1bf2fE3CcFe155B987f1D603188028" + "address": "protocol-state-contract" }, "callback_worker_config": { "num_snapshot_workers": 8, "num_aggregation_workers": 16 }, "ipfs": { - "url": "/dns/ipfs/tcp/5001", + "url": "ipfs-writer-url", "url_auth": { - "apiKey": "", - "apiSecret": "" + "apiKey": "ipfs-writer-key", + "apiSecret": "ipfs-writer-secret" }, - "reader_url": "/dns/ipfs/tcp/5001", + "reader_url": "ipfs-reader-url", "reader_url_auth": { - "apiKey": "", - "apiSecret": "" + "apiKey": "ipfs-reader-key", + "apiSecret": "ifps-reader-secret" }, "write_rate_limit": { "req_per_sec": 10, @@ -112,8 +112,8 @@ }, "anchor_chain_rpc": { "full_nodes": [{ - "url": "https://rpc-url", - "rate_limit": "10000000/day;2000/minute;100/second" + "url": "https://prost-rpc-url", + "rate_limit": "100000000/day;18000/minute;300/second" }], "retry": 5, "request_time_out": 5, diff --git a/snapshotter_autofill.sh b/snapshotter_autofill.sh index ebba6a41..3dda856e 100755 --- a/snapshotter_autofill.sh +++ b/snapshotter_autofill.sh @@ -1,46 +1,99 @@ #!/bin/bash #This script is run from high level docker-compose. Refer to https://github.com/PowerLoom/deploy +source .env -# TODO: Update this script according to new settings.json changes set -e echo 'populating setting from environment values...'; -if [ -z "$RPC_URL" ]; then +if [ -z "$SOURCE_RPC_URL" ]; then echo "RPC URL not found, please set this in your .env!"; exit 1; fi -if [ -z "$UUID" ]; then - echo "UUID not found, please set this in your .env!"; +if [ -z "$SIGNER_ACCOUNT_ADDRESS" ]; then + echo "SIGNER_ACCOUNT_ADDRESS not found, please set this in your .env!"; exit 1; fi -echo "Got RPC URL: ${RPC_URL}" +echo "Found SOURCE RPC URL ${SOURCE_RPC_URL}"; -echo "Got UUID: ${UUID}" +echo "Found SIGNER ACCOUNT ADDRESS ${SIGNER_ACCOUNT_ADDRESS}"; -echo "Got CONSENSUS_URL: ${CONSENSUS_URL}" +if [ "$PROST_RPC_URL" ]; then + echo "Found PROST_RPC_URL ${PROST_RPC_URL}"; +fi + +if [ "$IPFS_URL" ]; then + echo "Found IPFS_URL ${IPFS_URL}"; +fi + +if [ "$PROTOCOL_STATE_CONTRACT" ]; then + echo "Found PROTOCOL_STATE_CONTRACT ${PROTOCOL_STATE_CONTRACT}"; +fi + +if [ "$SLACK_REPORTING_URL" ]; then + echo "Found SLACK_REPORTING_URL ${SLACK_REPORTING_URL}"; +fi + +if [ "$POWERLOOM_REPORTING_URL" ]; then + echo "Found SLACK_REPORTING_URL ${POWERLOOM_REPORTING_URL}"; +fi + + +cp config/projects.example.json config/projects.json +cp config/aggregator.example.json config/aggregator.json +cp config/auth_settings.example.json config/auth_settings.json cp config/settings.example.json config/settings.json export namespace=UNISWAPV2-ph15-prod -export consensus_url="${CONSENSUS_URL:-https://offchain-consensus-api.powerloom.io}" +export prost_rpc_url="${PROST_RPC_URL:-https://rpc-prost1b.powerloom.io}" + +export ipfs_url="${IPFS_URL:-/dns/ipfs/tcp/5001}" +export ipfs_api_key="${IPFS_API_KEY:-}" +export ipfs_api_secret="${IPFS_API_SECRET:-}" + +export protocol_state_contract="${PROTOCOL_STATE_CONTRACT:-0xb71EAc336ffd776BAe4b1F861E58FaF13aB7c34B}" +export slack_reporting_url="${SLACK_REPORTING_URL:-}" +export powerloom_reporting_url="${POWERLOOM_REPORTING_URL:-}" + + + +# If IPFS_URL is empty, clear IPFS API key and secret +if [ -z "$IPFS_URL" ]; then + ipfs_api_key="" + ipfs_api_secret="" +fi echo "Using Namespace: ${namespace}" -echo "Using CONSENSUS_URL: ${consensus_url}" +echo "Using Prost RPC URL: ${prost_rpc_url}" +echo "Using IPFS URL: ${ipfs_url}" +echo "Using IPFS API KEY: ${ipfs_api_key}" +echo "Using protocol state contract: ${protocol_state_contract}" +echo "Using slack reporting url: ${slack_reporting_url}" +echo "Using powerloom reporting url: ${powerloom_reporting_url}" -sed -i "s|relevant-namespace|$namespace|" config/settings.json +sed -i'.bac' "s#relevant-namespace#$namespace#" config/settings.json -sed -i "s|https://rpc-url|$RPC_URL|" config/settings.json +sed -i'.backup' "s#account-address#$SIGNER_ACCOUNT_ADDRESS#" config/settings.json -sed -i "s|generated-uuid|$UUID|" config/settings.json +sed -i'.backup' "s#https://rpc-url#$SOURCE_RPC_URL#" config/settings.json -sed -i "s|https://consensus-url|$consensus_url|" config/settings.json +sed -i'.backup' "s#https://prost-rpc-url#$prost_rpc_url#" config/settings.json -cp config/auth_settings.example.json config/auth_settings.json +sed -i'.backup' "s#ipfs-writer-url#$ipfs_url#" config/settings.json +sed -i'.backup' "s#ipfs-writer-key#$ipfs_api_key#" config/settings.json +sed -i'.backup' "s#ipfs-writer-secret#$ipfs_api_secret#" config/settings.json -cp config/projects.example.json config/projects.json -cp config/aggregator.example.json config/aggregator.json +sed -i'.backup' "s#ipfs-reader-url#$ipfs_url#" config/settings.json +sed -i'.backup' "s#ipfs-reader-key#$ipfs_api_key#" config/settings.json +sed -i'.backup' "s#ipfs-reader-secret#$ipfs_api_secret#" config/settings.json + +sed -i'.backup' "s#protocol-state-contract#$protocol_state_contract#" config/settings.json + +sed -i'.backup' "s#https://slack-reporting-url#$slack_reporting_url#" config/settings.json + +sed -i'.backup' "s#https://powerloom-reporting-url#$powerloom_reporting_url#" config/settings.json echo 'settings has been populated!' From 752c608fe1f67bb724d5574f3d26bde247d08ff9 Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Mon, 29 May 2023 15:52:56 +0530 Subject: [PATCH 07/11] cleanup: remove .env source --- snapshotter_autofill.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/snapshotter_autofill.sh b/snapshotter_autofill.sh index 3dda856e..fd4aa0cc 100755 --- a/snapshotter_autofill.sh +++ b/snapshotter_autofill.sh @@ -1,8 +1,6 @@ #!/bin/bash #This script is run from high level docker-compose. Refer to https://github.com/PowerLoom/deploy -source .env - set -e echo 'populating setting from environment values...'; @@ -74,7 +72,7 @@ echo "Using protocol state contract: ${protocol_state_contract}" echo "Using slack reporting url: ${slack_reporting_url}" echo "Using powerloom reporting url: ${powerloom_reporting_url}" -sed -i'.bac' "s#relevant-namespace#$namespace#" config/settings.json +sed -i'.backup' "s#relevant-namespace#$namespace#" config/settings.json sed -i'.backup' "s#account-address#$SIGNER_ACCOUNT_ADDRESS#" config/settings.json From a6b94f20f2558fe962add9851e8b959d9b33b145 Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Mon, 29 May 2023 16:06:50 +0530 Subject: [PATCH 08/11] chore: minor typo fix in settings.example.json --- config/settings.example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/settings.example.json b/config/settings.example.json index cfb12968..ab385bdb 100644 --- a/config/settings.example.json +++ b/config/settings.example.json @@ -92,7 +92,7 @@ "reader_url": "ipfs-reader-url", "reader_url_auth": { "apiKey": "ipfs-reader-key", - "apiSecret": "ifps-reader-secret" + "apiSecret": "ipfs-reader-secret" }, "write_rate_limit": { "req_per_sec": 10, From 6bc23983c8e985ee4c1ad2dc01ccd70b2291e2e0 Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Mon, 29 May 2023 17:19:50 +0530 Subject: [PATCH 09/11] fix: make retry_state_callback sync --- pooler/utils/data_utils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pooler/utils/data_utils.py b/pooler/utils/data_utils.py index 893b95c6..5555ac99 100644 --- a/pooler/utils/data_utils.py +++ b/pooler/utils/data_utils.py @@ -24,7 +24,7 @@ logger = logger.bind(module='data_helper') -async def retry_state_callback(retry_state: tenacity.RetryCallState): +def retry_state_callback(retry_state: tenacity.RetryCallState): logger.warning(f'Encountered IPFS cat exception: {retry_state.outcome.exception()}') @@ -149,7 +149,7 @@ async def get_submission_data(redis_conn: aioredis.Redis, cid, ipfs_reader, proj try: submission_data = await fetch_file_from_ipfs(ipfs_reader, cid) except: - logger.error('Error while fetching data from IPFS | Project {} | CID {}', project_id, cid) + logger.error('Error while fetching data from IPFS | Project {} | CID {}', project_id, cid) submission_data = dict() else: # Cache it @@ -158,7 +158,12 @@ async def get_submission_data(redis_conn: aioredis.Redis, cid, ipfs_reader, proj return submission_data -async def get_sumbmission_data_bulk(redis_conn: aioredis.Redis, cids: List[str], ipfs_reader, project_ids: List[str]) -> List[dict]: +async def get_sumbmission_data_bulk( + redis_conn: aioredis.Redis, + cids: List[str], + ipfs_reader, + project_ids: List[str], +) -> List[dict]: batch_size = 10 all_snapshot_data = [] for i in range(0, len(cids), batch_size): From 844c5415e0a7f47660df56009b60acf3aed4ce85 Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Mon, 29 May 2023 17:47:07 +0530 Subject: [PATCH 10/11] chore: increase IPFS timeout --- config/settings.example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/settings.example.json b/config/settings.example.json index ab385bdb..e96d07cb 100644 --- a/config/settings.example.json +++ b/config/settings.example.json @@ -98,7 +98,7 @@ "req_per_sec": 10, "burst": 10 }, - "timeout": 20, + "timeout": 60, "local_cache_path": "/home/ubuntu/local_cache/", "connection_limits":{ "max_connections": 100, From da7f16daf24e04b87811e354ac721eddea1d6e48 Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Wed, 31 May 2023 15:36:10 +0530 Subject: [PATCH 11/11] chore: change upload everything to web3storage to false --- config/settings.example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/settings.example.json b/config/settings.example.json index e96d07cb..1ce1c1a0 100644 --- a/config/settings.example.json +++ b/config/settings.example.json @@ -107,7 +107,7 @@ } }, "web3storage":{ - "upload_snapshots": true, + "upload_snapshots": false, "upload_aggregates": true }, "anchor_chain_rpc": {