various small changes

.env.sample

@@ -28,3 +28,21 @@ ETHERSCAN_API_KEY=
 
 # Query ID for the aggregate query on Dune
 AGGREGATE_QUERY_ID=
+
+# Node for each chain we want to run a sync job on
+NODE_URL_MAINNET=
+NODE_URL_GNOSIS=
+NODE_URL_ARBITRUM=
+
+# The network which we run a sync job on.
+# Current options are: {"mainnet", "xdai", "arbitrum-one"}
+NETWORK=
+
+# The prefix of the dune table where we sync the various mainnet price feeds
+PRICE_FEED_TARGET_TABLE
+
+# The prefix of the dune table where we sync the monthly raw batch data
+BATCH_DATA_TARGET_TABLE = "batch_data"
+
+# Dune api timeout parameter, where we recommend to set it to 600
+DUNE_API_REQUEST_TIMEOUT = 600

src/fetch/orderbook.py

@@ -19,6 +19,7 @@
 log = set_log(__name__)
 
 MAX_PROCESSING_DELAY = 10
+BUCKET_SIZE = {"mainnet": 10000, "xdai": 30000, "arbitrum-one": 1000000}
 
 
 class OrderbookEnv(Enum):
@@ -229,9 +230,10 @@ def get_batch_data(cls, block_range: BlockRange) -> DataFrame:
         so as to ensure the batch data query runs fast enough.
         At the end, it concatenates everything into one data frame
         """
+        load_dotenv()
         start = block_range.block_from
         end = block_range.block_to
-        bucket_size = 20000
+        bucket_size = BUCKET_SIZE[os.environ.get("NETWORK", "mainnet")]
         res = []
         while start < end:
             size = min(end - start, bucket_size)

src/main.py

@@ -23,6 +23,7 @@
 )
 from src.sync.order_rewards import sync_order_rewards, sync_batch_rewards
 from src.sync.batch_data import sync_batch_data
+from src.sync.common import node_suffix
 
 log = set_log(__name__)
 
@@ -66,15 +67,9 @@ def main() -> None:
     orderbook = OrderbookFetcher()
     network = os.environ.get("NETWORK", "mainnet")
     log.info(f"Network is set to: {network}")
-    if network == "mainnet":
-        node_suffix = "MAINNET"
-    else:
-        if network == "xdai":
-            node_suffix = "GNOSIS"
-        else:
-            if network == "arbitrum-one":
-                node_suffix = "ARBITRUM"
-    web3 = Web3(Web3.HTTPProvider(os.environ.get("NODE_URL" + "_" + node_suffix)))
+    web3 = Web3(
+        Web3.HTTPProvider(os.environ.get("NODE_URL" + "_" + node_suffix(network)))
+    )
 
     if args.sync_table == SyncTable.APP_DATA:
         table = os.environ["APP_DATA_TARGET_TABLE"]

src/sync/batch_data.py

@@ -1,11 +1,12 @@
 """Main Entry point for batch data sync"""
 import os
+from dotenv import load_dotenv
 from dune_client.client import DuneClient
 import web3
 from src.fetch.orderbook import OrderbookFetcher
 from src.logger import set_log
 from src.sync.config import BatchDataSyncConfig
-from src.sync.common import compute_block_and_month_range
+from src.sync.common import compute_block_and_month_range, node_suffix
 from src.models.block_range import BlockRange
 
 
@@ -20,15 +21,9 @@ async def sync_batch_data(
     dry_run: bool,
 ) -> None:
     """Batch data Sync Logic"""
+    load_dotenv()
     network = os.environ["NETWORK"]
-
-    if network == "mainnet":
-        network_name = "ethereum"
-    else:
-        if network == "xdai":
-            network_name = "gnosis"
-        else:
-            network_name = "arbitrum"
+    network_name = node_suffix(network).lower()
 
     block_range_list, months_list = compute_block_and_month_range(node)
     for i in range(len(block_range_list)):

src/sync/common.py

@@ -21,6 +21,18 @@ def last_sync_block(aws: AWSClient, table: SyncTable, genesis_block: int = 0) ->
     return block_from
 
 
+def node_suffix(network: str) -> str:
+    if network == "mainnet":
+        return "MAINNET"
+    else:
+        if network == "xdai":
+            return "GNOSIS"
+        else:
+            if network == "arbitrum-one":
+                return "ARBITRUM"
+    return ""
+
+
 def find_block_with_timestamp(node, time_stamp) -> int:
     """
     This implements binary search and returns the smallest block number