chore: submodules init for phase 2

PowerLoom · Nov 14, 2023 · 991015f · 991015f
1 parent 2340a91
commit 991015f
Show file tree

Hide file tree

Showing 38 changed files with 99 additions and 5,677 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,6 @@
+[submodule "config"]
+	path = config
+	url = https://github.com/PowerLoom/snapshotter-phase2-configs
+[submodule "snapshotter/modules/boost"]
+	path = snapshotter/modules/boost
+	url = https://github.com/PowerLoom/snapshotter-phase2-computes
diff --git a/config/.gitignore b/config/.gitignore
@@ -0,0 +1,11 @@
+.env
+.python-version
+__pycache__/
+.idea/
+.DS_Store
+screenlog*
+temp/*
+*projects.json
+*aggregator.json
+*settings.json
+*.backup
diff --git a/config/aggregator.example.json b/config/aggregator.example.json
@@ -1,114 +1,3 @@
 {
-  "config": [{
-      "project_type": "aggregate_pairContract_24h_trade_volume",
-      "aggregate_on": "SingleProject",
-      "filters": {
-        "projectId": "pairContract_trade_volume"
-      },
-      "processor": {
-        "module": "snapshotter.modules.pooler.uniswapv2.aggregate.single_uniswap_trade_volume_24h",
-        "class_name": "AggreagateTradeVolumeProcessor"
-      }
-    },
-    {
-      "project_type": "aggregate_pairContract_7d_trade_volume",
-      "aggregate_on": "SingleProject",
-      "filters": {
-        "projectId": "aggregate_pairContract_24h_trade_volume"
-      },
-      "processor": {
-        "module": "snapshotter.modules.pooler.uniswapv2.aggregate.single_uniswap_trade_volume_7d",
-        "class_name": "AggreagateTradeVolumeProcessor"
-      }
-    },
-    {
-      "project_type": "aggregate_24h_top_pairs_lite",
-      "aggregate_on": "MultiProject",
-      "projects_to_wait_for": [
-        "aggregate_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2",
-        "pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2",
-        "pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2"
-      ],
-      "processor": {
-        "module": "snapshotter.modules.pooler.uniswapv2.aggregate.multi_uniswap_top_pairs_24h",
-        "class_name": "AggreagateTopPairsProcessor"
-      }
-    },
-    {
-      "project_type": "aggregate_24h_top_tokens_lite",
-      "aggregate_on": "MultiProject",
-      "projects_to_wait_for": [
-        "aggregate_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2",
-        "pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2",
-        "pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2"
-      ],
-      "processor": {
-        "module": "snapshotter.modules.pooler.uniswapv2.aggregate.multi_uniswap_top_tokens",
-        "class_name": "AggreagateTopTokensProcessor"
-      }
-    },
-    {
-      "project_type": "aggregate_7d_top_pairs_lite",
-      "aggregate_on": "MultiProject",
-      "projects_to_wait_for": [
-        "aggregate_pairContract_7d_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2",
-        "aggregate_pairContract_7d_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2",
-        "aggregate_pairContract_7d_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2",
-        "aggregate_pairContract_7d_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2",
-        "aggregate_pairContract_7d_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2",
-        "aggregate_pairContract_7d_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2",
-        "aggregate_pairContract_7d_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2"
-      ],
-      "processor": {
-        "module": "snapshotter.modules.pooler.uniswapv2.aggregate.multi_uniswap_top_pairs_7d",
-        "class_name": "AggreagateTopPairsProcessor"
-      }
-    },
-    {
-      "project_type": "aggregate_24h_stats_lite",
-      "aggregate_on": "MultiProject",
-      "projects_to_wait_for": [
-        "aggregate_pairContract_24h_trade_volume:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2",
-        "pairContract_pair_total_reserves:0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2",
-        "pairContract_pair_total_reserves:0x3041cbd36888becc7bbcbc0045e3b1f144466f5f:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xd3d2e2692501a5c9ca623199d38826e513033a17:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xbb2b8038a1640196fbe3e38816f3e67cba72d940:UNISWAPV2",
-        "aggregate_pairContract_24h_trade_volume:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2",
-        "pairContract_pair_total_reserves:0xa478c2975ab1ea89e8196811f51a7b7ade33eb11:UNISWAPV2"
-      ],
-      "processor": {
-        "module": "snapshotter.modules.pooler.uniswapv2.aggregate.multi_uniswap_stats",
-        "class_name": "AggreagateStatsProcessor"
-      }
-    }
-  ]
+  "config": []
 }
diff --git a/config/preloader.json b/config/preloader.json
@@ -23,5 +23,5 @@
           "class_name": "TxReceiptProcessor"
         }
     ],
-    "timeout": 60
+    "timeout": 90
 }
diff --git a/config/projects.example.json b/config/projects.example.json
@@ -1,43 +1,88 @@
 {
-  "config": [{
-      "project_type": "pairContract_pair_total_reserves",
+  "config": [
+    {
+      "project_type": "zkevm:bungee_bridge",
+      "projects":[
+        ],
       "preload_tasks":[
-        "eth_price",
-        "block_details"
+        "block_transactions"
       ],
+      "processor":{
+        "module": "snapshotter.modules.boost.bungee_bridge",
+        "class_name": "BungeeBridgeProcessor"
+      }
+    },
+    {
+      "project_type": "zkevm:owlto_bridge",
       "projects":[
-        "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc",
-        "0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5",
-        "0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852",
-        "0x3041cbd36888becc7bbcbc0045e3b1f144466f5f",
-        "0xd3d2e2692501a5c9ca623199d38826e513033a17",
-        "0xbb2b8038a1640196fbe3e38816f3e67cba72d940",
-        "0xa478c2975ab1ea89e8196811f51a7b7ade33eb11"
         ],
+      "preload_tasks":[
+        "block_transactions"
+      ],
       "processor":{
-        "module": "snapshotter.modules.pooler.uniswapv2.pair_total_reserves",
-        "class_name": "PairTotalReservesProcessor"
+        "module": "snapshotter.modules.boost.owlto_bridge",
+        "class_name": "OwltoBridgeProcessor"
       }
     },
     {
-      "project_type": "pairContract_trade_volume",
+      "project_type": "zkevm:quickswap_usdc_swap",
+      "projects":[
+        ],
       "preload_tasks":[
-        "eth_price",
-        "block_details"
+        "block_transactions"
       ],
+      "processor":{
+        "module": "snapshotter.modules.boost.quickswap_usdc_swap",
+        "class_name": "QuickswapUSDCSwapProcessor"
+      }
+    },
+    {
+      "project_type": "zkevm:quickswap_dai_swap",
       "projects":[
-        "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc",
-        "0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5",
-        "0x0d4a11d5eeaac28ec3f61d100daf4d40471f1852",
-        "0x3041cbd36888becc7bbcbc0045e3b1f144466f5f",
-        "0xd3d2e2692501a5c9ca623199d38826e513033a17",
-        "0xbb2b8038a1640196fbe3e38816f3e67cba72d940",
-        "0xa478c2975ab1ea89e8196811f51a7b7ade33eb11"
         ],
-        "processor":{
-          "module": "snapshotter.modules.pooler.uniswapv2.trade_volume",
-          "class_name": "TradeVolumeProcessor"
-        }
+      "preload_tasks":[
+        "block_transactions"
+      ],
+      "processor":{
+        "module": "snapshotter.modules.boost.quickswap_dai_swap",
+        "class_name": "QuickswapDAISwapProcessor"
+      }
+    },
+    {
+      "project_type": "zkevm:quickswap_eth_usdc_lp",
+      "projects":[
+        ],
+      "preload_tasks":[
+        "block_transactions"
+      ],
+      "processor":{
+        "module": "snapshotter.modules.boost.quickswap_eth_usdc_lp",
+        "class_name": "QuickswapEthUSDCLPProcessor"
+      }
+    },
+    {
+      "project_type": "zkevm:safe_create",
+      "projects":[
+        ],
+      "preload_tasks":[
+        "block_transactions"
+      ],
+      "processor":{
+        "module": "snapshotter.modules.boost.safe_create",
+        "class_name": "SafeCreateProcessor"
+      }
+    },
+    {
+      "project_type": "zkevm:safe_add_owner",
+      "projects":[
+        ],
+      "preload_tasks":[
+        "block_transactions"
+      ],
+      "processor":{
+        "module": "snapshotter.modules.boost.safe_add_owner",
+        "class_name": "SafeAddOwnerProcessor"
+      }
     }
   ]
 }
diff --git a/config/settings.example.json b/config/settings.example.json
@@ -13,14 +13,14 @@
   "rpc": {
     "full_nodes": [{
       "url": "https://rpc-url",
-      "rate_limit": "100000000/day;20000/minute;1000/second"
+      "rate_limit": "10000000/day;2000/minute;100/second"
     }],
     "archive_nodes": [],
     "force_archive_blocks": 100,
     "retry": 5,
     "request_time_out": 5,
     "skip_epoch_threshold_blocks": 30,
-    "polling_interval": 10,
+    "polling_interval": 15,
     "connection_limits":{
       "max_connections": 100,
       "max_keepalive_connections": 50,
@@ -84,9 +84,9 @@
     "address": "protocol-state-contract"
   },
   "callback_worker_config": {
-    "num_delegate_workers": 0,
-    "num_snapshot_workers": 8,
-    "num_aggregation_workers": 12
+    "num_delegate_workers": 8,
+    "num_snapshot_workers": 2,
+    "num_aggregation_workers": 4
   },
   "ipfs": {
     "url": "ipfs-writer-url",
@@ -109,29 +109,16 @@
       "max_connections": 100,
       "max_keepalive_connections": 50,
       "keepalive_expiry": 300
-    },
-    "remote_pinning": {
-      "enabled": false,
-      "service_name": "",
-      "service_endpoint": "",
-      "service_token": "",
-      "background_pinning": false
     }
   },
   "web3storage":{
     "upload_snapshots": false,
-    "upload_aggregates": true,
-    "url": "https://api.web3.storage",
-    "upload_url_suffix": "/upload",
-    "api_token": "web3-storage-token",
-    "timeout": 30,
-    "max_idle_conns": 5,
-    "idle_conn_timeout": 300
+    "upload_aggregates": true
   },
   "anchor_chain_rpc": {
     "full_nodes": [{
       "url": "https://prost-rpc-url",
-      "rate_limit": "100000000/day;20000/minute;1000/second"
+      "rate_limit": "100000000/day;18000/minute;300/second"
     }],
     "retry": 5,
     "request_time_out": 5,

diff --git a/snapshotter/modules/boost b/snapshotter/modules/boost
diff --git a/snapshotter/modules/pooler/uniswapv2/__init__.py b/snapshotter/modules/pooler/uniswapv2/__init__.py
diff --git a/snapshotter/modules/pooler/uniswapv2/aggregate/__init__.py b/snapshotter/modules/pooler/uniswapv2/aggregate/__init__.py