Merge pull request #51 from itzmeanjan/develop

Develop

.gitignore

@@ -17,3 +17,4 @@
 ette
 node_modules
 .plans.json
+snapshot.bin

Makefile

@@ -0,0 +1,18 @@
+proto_clean:
+	rm -rfv app/pb
+
+proto_gen:
+	mkdir app/pb
+	protoc -I app/proto/ --go_out=paths=source_relative:app/pb app/proto/*.proto
+
+graphql_gen:
+	pushd app/rest
+	gqlgen generate
+	popd
+
+build:
+	go build -o ette
+
+run:
+	go build -o ette
+	./ette

README.md

@@ -27,6 +27,9 @@ EVM-based Blockchain Indexer, with historical data query & real-time notificatio
         - [Real-time block mining notification](#real-time-notification-for-mined-blocks-)
         - [Real-time transaction notification ( 🤩 Filters Added ) ](#real-time-notification-for-transactions-%EF%B8%8F)
         - [Real-time log event notification ( 🤩 Filters Added ) ](#real-time-notification-for-events-)
+    - Snapshotting
+        - [Take snapshot](#take-snapshot-of-existing-data-store-%EF%B8%8F)
+        - [Restore from snapshot](#restore-data-from-snapshot-%EF%B8%8F)
 
 ## Inspiration 🤔
 
@@ -50,11 +53,23 @@ It's not that I was unable find any solution, but wasn't fully satisfied with th
 - All historical data query requests must carry authentication header as `APIKey`, which can be generated by users, using webUI, packed with `ette`.
 - All real-time event subscription & unsubscription requests must carry `apiKey` in their payload.
 - It has very minimalistic webUI for creating & managing `APIKey`(s).
+- It has capability to process blocks in delayed fashion, if asked to do so. **To address chain reorganization issue, this is very effective**. All you need to do, specify how many block confirmations you require before considering that block to be finalized in `.env` file. Now `ette` will do everything with block _( if real-time subscription mode is enabled, it'll publish data to clients who're interested i.e. subscribed )_ expect putting it in persistent data store. Rather block identifier to be put in waiting queue, from where it'll be eventually picked up by workers to finally persist it in DB. Only downside of using this feature is you might not get data back in response of query for certain block number, which just got mined but not finalized as per your set up i.e. `BlockConfirmations` environment variable's value. You can always skip it, default value will be **0**.
+
+- `ette` can help you in taking snapshot of whole database, it's relying on, into a single binary file, where block data is serialized into Protocol Buffer format, efficient for deserialization also i.e. while restoring back from snapshot.
+    - `EtteMode` = 4, attempts to take a snapshot of whole database.
+
+- Restoring from snapshoted data file, can be attempted by `ette` when `EtteMode` = 5. Make sure you've cleaned backing data store before so & recreated database. [ **Table migration to be automatically taken care of** ]
+
+- For snapshotting purposes, you can always set sink/ source data file in `SnapshotFile` in `.env`.
+
+- 👆 snapshotting feature is helpful, if you're willing migrate `ette` to different machine or setting up new instance of `ette`. If you want to avoid a lengthy whole chain data syncing, you must take snapshot from existing instance of `ette` & attempt to restore from binary snapshot file in new `ette` instance.
 
 And that's `ette`
 
 ## Prerequisite 👍
 
+![running_ette](./sc/running-ette.png)
+
 - Make sure you've Go _( >= 1.15 )_ installed
 - You need to also install & set up PostgreSQL. I found [this](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-20-04) guide helpful.
 - Redis needs to be installed too. Consider following [this](https://www.digitalocean.com/community/tutorials/how-to-install-and-secure-redis-on-ubuntu-20-04) guide.
@@ -82,17 +97,19 @@ cd ette
 - Create a `.env` file in this directory. 
 
     - Make sure PostgreSQL has md5 authentication mechanism enabled.
-    - Please enable password based authentication in Redis Server.
-    - `Admin` is ethereum address who will be able to administer `ette` using webUI. _[ **This feature is not yet implemented, but please put `Admin` in config file** ]_
+    - Please enable password based authentication in Redis Server
+    - Skipping `RedisPassword` is absolutely fine, if you don't want to use any password in Redis instance. [ **Not recommended** ]
     - Replace `Domain` with your domain name i.e. `ette.company.com`
     - Set `Production` to `yes` before running it in production; otherwise you can simply skip it
-    - `ette` can be run in any of 3 possible modes, which can be set by `EtteMode`
+    - `ette` can be run in any of 5 possible modes, which can be set by `EtteMode`
 
     ```json
     {
         "1": "Only Historical Data Query Allowed",
-        "2": "Only Real-time Event Subscription Allowed",
-        "3": "Both Historical Data Query & Real-time Event Subscription Allowed"
+        "2": "Only Real-time Subscription Allowed",
+        "3": "Both Historical Data Query & Real-time Subscription Allowed",
+        "4": "Attempt to take snapshot from data in backing DB",
+        "5": "Attempt to restore data from snapshot file"
     }
     ```
 
@@ -102,10 +119,10 @@ cd ette
     - 👆 being done for controlling concurrency level, by putting more control on user's hand.
     - If you want to persist blocks in delayed fashion, you might consider setting `BlockConfirmations` to some _number > 0_.
     - That will make `ette` think you're asking it 80 is latest block, which can be persisted in final data store, when latest mined block number is 100 & `BlockConfirmations` is set to 20.
-    - This option is **recommended** to be used, at least in production.
-    - Skipping `RedisPassword` is absolutely fine, if you don't want to use any password in Redis instance. [ **Not recommended** ]
+    - This option is **recommended** to be used, at least in production, to address _chain reorganization issue_.
     - For range based queries `BlockRange` can be set to limit how many blocks can be queried by client in a single go. Default value 100.
     - For time span based queries `TimeRange` can be set to put limit on max time span _( in terms of second )_, can be used by clients. Default value 3600 i.e. 1 hour.
+    - If you're attempting to take snapshot/ restore from binary snapshot file, you can set `SnapshotFile` in `.env` file, to set sink/ source file name, respectively. Default file name `echo $(echo $(pwd)/snapshot.bin)` in i.e. from where `ette` gets invoked. Consider setting `EtteMode` correctly, depending upon what you want to attain.
 
 ```
 RPCUrl=https://<domain-name>
@@ -119,15 +136,15 @@ DB_NAME=ette
 RedisConnection=tcp
 RedisAddress=x.x.x.x:6379
 RedisPassword=password
-Admin=e19b9EB3Bf05F1C8100C9b6E8a3D8A14F6384BFb
 Domain=localhost
 Production=yes
 EtteMode=3
 EtteGraphQLPlayGround=yes
-ConcurrencyFactor=2
-BlockConfirmations=20
+ConcurrencyFactor=5
+BlockConfirmations=200
 BlockRange=1000
 TimeRange=21600
+SnapshotFile=snapshot.bin
 ```
 
 - Create another file in same directory, named `.plans.json`, whose content will look like 👇.
@@ -169,13 +186,16 @@ TimeRange=21600
 - Now build `ette`
 
 ```bash
-go build
+make build
 ```
 
 - If everything goes as expected, you'll find one binary named, **ette** in this directory. Run it. 
 
 ```bash
 ./ette
+
+# or directly run `ette` using 👇, which will first build, then run
+make run
 ```
 
 - Database migration to be taken care of during application start up.
@@ -196,15 +216,85 @@ curl -s localhost:7000/v1/synced | jq
 }
 ```
 
-> Note: For production, you'll most probably run it using `systemd`
+---
+
+### Production deployment of `ette` using **systemd**
+
+Here's a systemd unit file which you can create in `/etc/systemd/system`.
+
+```bash
+sudo touch /etc/systemd/system/ette.service # first do it
+```
+
+Now you can paste 👇 content in unit file, given that you've cloned `ette` in **$HOME**.
+
+```bash
+[Unit]
+Description=ette - EVM Blockchain Indexer
+
+[Service]
+User=ubuntu
+WorkingDirectory=/home/ubuntu/ette
+ExecStart=/home/ubuntu/ette/ette
+Restart=on-failure
+RestartSec=10s
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Time to load systemd.
+
+```bash
+sudo systemctl daemon-reload
+```
+
+Now you can enable `ette`, so that it can be automatically started after system boot up.
+
+```bash
+sudo systemctl enable ette.service
+```
+
+Finally you can start `ette`.
+
+```bash
+sudo systemctl start ette.service
+```
+
+You can also stop, running `ette` instance.
+
+```bash
+sudo systemctl stop ette.service
+```
+
+Restart an instance.
+
+```bash
+sudo systemctl restart ette.service
+```
+
+All logs `ette` produces can be inspected using 👇
+
+```bash
+sudo journalctl -u ette.service # oldest to newest
+sudo journalctl -u ette.service --reverse # opposite of 👆
+```
+
+Latest log can be followed
+
+```bash
+sudo journalctl -u ette.service -f
+```
+
+---
 
 ## Use Cases 🤯
 
 `ette` is supposed to be deployed by anyone, interested in running a historical data query & real-time notification service for EVM-based blockchain(s).
 
-All client requests are rate limited _( 50k requests/ day )_. This rate limit is enforced on all `APIKey`(s) created by any single Ethereum Address. You can create multiple `APIKey`(s) from your account & accumulated requests made from those keys to be considered before dropping your requests.
+All client requests are by default rate limited _( 50k requests/ day )_. This rate limit is enforced on all `APIKey`(s) created by any single Ethereum Address. You can create multiple `APIKey`(s) from your account & accumulated requests made from those keys to be considered before dropping your requests.
 
-> So, it'll be helpful for protecting from spam attacks.
+If you need more requests per day, you can always asked your `ette` administrator to manually increase that from database table. _[ **Risky operation, needs to be done carefully. This is not recommended.** ]_
 
 **More features coming here, soon**
 
@@ -715,4 +805,20 @@ You'll receive 👇 response, confirming unsubscription
 
 > Note: If graceful unsubscription not done, when `ette` finds client unreachable, it'll remove client subscription
 
+### Take snapshot of existing data store ➡️
+
+Assuming you've already a running instance of `ette` for some EVM compatible chain, you can always attempt to take snapshot of whole backing data store, so that if you need to spin up another instance of `ette`, you won't require to sync whole chain data, rather you use this binary data file, which can be used by `ette` for restoring from snapshot data.
+
+Setting `EtteMode` = 4, attempts to take snapshot of DB. 
+
+![taking-snapshot](./sc/taking-snapshot.png)
+
+### Restore data from snapshot ⬅️
+
+Once you've snapshotted binary encoded data file, you can attempt to restore from this & rebuild whole data store, with out syncing whole chain data. `EtteMode` = 5, attempts to do 👇.
+
+![restoring-from-snapshot](./sc/restoring-from-snapshot.png)
+
+Once that's done, consider restarting `ette` in desired mode so that it can keep itself in sync with latest chain happenings.
+
 **More coming soon**

app/app.go

@@ -6,6 +6,8 @@ import (
 	"os"
 	"os/signal"
 	"sync"
+	"syscall"
+	"time"
 
 	"github.com/go-redis/redis/v8"
 	"github.com/gookit/color"
@@ -15,6 +17,7 @@ import (
 	"github.com/itzmeanjan/ette/app/db"
 	"github.com/itzmeanjan/ette/app/rest"
 	"github.com/itzmeanjan/ette/app/rest/graph"
+	ss "github.com/itzmeanjan/ette/app/snapshot"
 	"gorm.io/gorm"
 )
 
@@ -25,7 +28,7 @@ func bootstrap(configFile, subscriptionPlansFile string) (*d.BlockChainNodeConne
 		log.Fatalf("[!] Failed to read `.env` : %s\n", err.Error())
 	}
 
-	if !(cfg.Get("EtteMode") == "1" || cfg.Get("EtteMode") == "2" || cfg.Get("EtteMode") == "3") {
+	if !(cfg.Get("EtteMode") == "1" || cfg.Get("EtteMode") == "2" || cfg.Get("EtteMode") == "3" || cfg.Get("EtteMode") == "4" || cfg.Get("EtteMode") == "5") {
 		log.Fatalf("[!] Failed to find `EtteMode` in configuration file\n")
 	}
 
@@ -75,7 +78,7 @@ func Run(configFile, subscriptionPlansFile string) {
 	// Attempting to listen to Ctrl+C signal
 	// and when received gracefully shutting down `ette`
 	interruptChan := make(chan os.Signal, 1)
-	signal.Notify(interruptChan, os.Interrupt)
+	signal.Notify(interruptChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL)
 
 	// All resources being used gets cleaned up
 	// when we're returning from this function scope
@@ -105,6 +108,51 @@ func Run(configFile, subscriptionPlansFile string) {
 
 	}()
 
+	// User has requested `ette` to take a snapshot of current database state
+	if cfg.Get("EtteMode") == "4" {
+
+		// checking if there's anything to snapshot or not
+		if _status.BlockCountInDB() == 0 {
+			log.Printf("[*] Nothing to snapshot\n")
+			return
+		}
+
+		// this is the file snapshot to be taken
+		_snapshotFile := cfg.GetSnapshotFile()
+		_start := time.Now().UTC()
+
+		log.Printf("[*] Starting snapshotting at : %s [ Sink : %s ]\n", _start, _snapshotFile)
+
+		// taking snapshot, this might take some time
+		_ret := ss.TakeSnapshot(_db, _snapshotFile, db.GetCurrentOldestBlockNumber(_db), db.GetCurrentBlockNumber(_db), _status.BlockCountInDB())
+		if _ret {
+			log.Printf(color.Green.Sprintf("[+] Snapshotted in : %s [ Count : %d ]", time.Now().UTC().Sub(_start), _status.BlockCountInDB()))
+		} else {
+			log.Printf(color.Red.Sprintf("[!] Snapshotting failed in : %s", time.Now().UTC().Sub(_start)))
+		}
+
+		return
+
+	}
+
+	// User has asked `ette` to attempt to restore from snapshotted data
+	// where data file is `snapshot.bin` in current working directory,
+	// if nothing specified for `SnapshotFile` variable in `.env`
+	if cfg.Get("EtteMode") == "5" {
+
+		_snapshotFile := cfg.GetSnapshotFile()
+		_start := time.Now().UTC()
+
+		log.Printf("[*] Starting snapshot restoring at : %s [ Sink : %s ]\n", _start, _snapshotFile)
+
+		_, _count := ss.RestoreFromSnapshot(_db, _snapshotFile)
+
+		log.Printf(color.Green.Sprintf("[+] Restored from snapshot in : %s [ Count : %d ]", time.Now().UTC().Sub(_start), _count))
+
+		return
+
+	}
+
 	// Pushing block header propagation listener to another thread of execution
 	go blk.SubscribeToNewBlocks(_connection, _db, _status, &_redisInfo)