From 680a2ced5d554309257ac384136c1159fe6587fe Mon Sep 17 00:00:00 2001 From: charmful0x Date: Fri, 27 Sep 2024 11:17:16 +0200 Subject: [PATCH 1/2] chore: change genesis start --- src/utils/backfill_genesis.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/backfill_genesis.rs b/src/utils/backfill_genesis.rs index 8202b56..b295f3f 100644 --- a/src/utils/backfill_genesis.rs +++ b/src/utils/backfill_genesis.rs @@ -6,7 +6,7 @@ use anyhow::{Error, Ok}; pub async fn backfill_from_genesis() -> Result<(), Error> { let network = Network::config(); let config_start_block = network.start_block; - let backfill_blocks: Vec = (0..=config_start_block).collect(); + let backfill_blocks: Vec = (3719..=config_start_block).collect(); if config_start_block == 0 { return Ok(()); From 28808e0f706e5c9b65d8eea1e4dd4c71a086125c Mon Sep 17 00:00:00 2001 From: charmful0x Date: Fri, 27 Sep 2024 15:09:39 +0200 Subject: [PATCH 2/2] feat: dynamic backfill start --- .env.example | 1 + README.md | 10 ++++++++++ src/utils/backfill_genesis.rs | 4 +++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 6d8ec66..c237a14 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,6 @@ archiver_pk="..." backfill_pk="..." +backfill_start_block="0" network="./networks/your_network.json" DATABASE_HOST="" diff --git a/README.md b/README.md index db03ef7..df3ce55 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ While a WeaveVM Archiver node can run without web2 component dependencies, this ```js archiver_pk="" // WeaveVM archiver PK backfill_pk="" // WeaveVM backfill PK +backfill_start_block="0" // it defaults to 0 (genesis), but it's dynamic, so you can specify from which block number you want to start backfilling network="./networks/your_network.json" DATABASE_HOST="" // planetscale @@ -52,6 +53,15 @@ To start archiving your network block data on WeaveVM: 6. set `start_block` value to the most recent network's blockheight. That will facilitate the archiver to start in sync with live blockheight while, in parallel, reindexing from genesis using the `backfill_address`. 7. Set up your PlanetScale DB according to `db_schema.sql`. +#### Parallel Threads of Archiving + +As mentioned previously, `archiver_address` is responsible for archiving blocks starting from the `start_block` specified in your `network.json` config file, while also keeping up with the network’s current blockheight (live sync). Meanwhile, `backfill_address` handles archiving blocks from `backfill_start_block` up to `start_block`. + +```txt +backfill thread: backfill_start_block -> start_block +live sync thread: start_block -> network's live blockheight +``` + ### RPC Proxy and Caching You can use [eRPC](https://github.com/erpc/erpc) to cache, load-balance and failover between as many RPC endpoints and use eRPC's proxy URL in each network's config for WeaveVM. This will increase performance and resiliency and reduce RPC usage cost while fetching network's block data via WeaveVM. diff --git a/src/utils/backfill_genesis.rs b/src/utils/backfill_genesis.rs index b295f3f..f5072be 100644 --- a/src/utils/backfill_genesis.rs +++ b/src/utils/backfill_genesis.rs @@ -1,12 +1,14 @@ use crate::utils::archive_block::archive; use crate::utils::planetscale::ps_archive_block; use crate::utils::schema::Network; +use crate::utils::env_var::get_env_var; use anyhow::{Error, Ok}; pub async fn backfill_from_genesis() -> Result<(), Error> { let network = Network::config(); let config_start_block = network.start_block; - let backfill_blocks: Vec = (3719..=config_start_block).collect(); + let backfill_block_start: String = get_env_var("backfill_start_block").unwrap_or(0.to_string()); + let backfill_blocks: Vec = (backfill_block_start.parse::().unwrap()..=config_start_block).collect(); if config_start_block == 0 { return Ok(());