Improve debugging and CLI test setup

README.md

@@ -189,24 +189,24 @@ needs to reach the tip before submitting transactions. The data checks will
 take a while to complete as well (likely a couple of days on a normal laptop
 with the current settings) as they reconcile balances for the entire chain._
 
-- Install the [`rosetta-cli`](https://github.com/coinbase/rosetta-cli) according to the instructions. (Note that on Mac, installing the `rosetta-cli` to `/usr/local/bin` or adding its location to you `$PATH` will allow you to call `rosetta-cli` directly on the command line rather than needing to provide the path to the executable). Current testing has been done with `v0.5.16` of the `rosetta-cli`.
-- Run the Rosetta service in the background for the respective network (currently only alfajores for both Data and Construction checks)
-- Run the CLI checks for alfajores as follows:
+- Install the [`rosetta-cli`](https://github.com/coinbase/rosetta-cli) according to the instructions. (Note that on Mac, installing the `rosetta-cli` to `/usr/local/bin` or adding its location to you `$PATH` will allow you to call `rosetta-cli` directly on the command line rather than needing to provide the path to the executable). Current testing has been done with `v0.10.3` of the `rosetta-cli`.
+- Run the Rosetta service in the background for the respective network (currently only alfajores for both data and construction checks, the data checks work for all other networks).
+- Run the CLI checks for `NETWORK_NAME` as follows:
 
 ```sh
 # alfajores; specify construction or data
-rosetta-cli check:construction --configuration-file PATH/TO/rosetta/rosetta-cli-conf/testnet/cli-config.json
+rosetta-cli check:data --configuration-file $PATH_TO_ROSETTA/rosetta-cli-conf/$NETWORK_NAME/cli-config.json
 ```
 
+[See below](#running-rosetta-with-a-mycelo-testnet) for more details on running the reconciliation checks against a mycelo testnet.
 
 ### How to generate `bootstrap_balances.json`
 
-This is only necessary for running the data checks if it has not already been created for the particular network. Here's how to generate this for alfajores (for another network, specify the appropriate genesis block URL and output path):
+This is only necessary for running the data checks if it has not already been created for the particular network. Here's how to generate this for alfajores (for another network, specify the appropriate genesis block filepath and output path):
 
 ```sh
 go run examples/generate_balances/main.go \
-  https://storage.googleapis.com/genesis_blocks/alfajores \
-  rosetta-cli-conf/testnet/bootstrap_balances.json
+  $PATH_TO_GENESIS_FILE rosetta-cli-conf/mycelo/bootstrap_balances.json
 ```
 
 ### Running Rosetta with a mycelo testnet
@@ -215,11 +215,11 @@ go run examples/generate_balances/main.go \
 - Set `--geth.networkid` to the network ID (if this is a deployed testnet, this may be different from the `ChainID` in the genesis file). If this value is the same as the `ChainID`, it is not necessary to set this parameter.
 - Set the `--monitor.initcontracts` flag (at least on the first run), which fetches necessary state from the genesis block and updates the Rosetta DB accordingly.
 
-To run reconciliation tests on this network:
+To run reconciliation tests on this network, you can use the `cli-config` in `./rosetta-cli-conf/mycelo` as a basis. (Note: the default `cli-config` uses the chain ID that is set when the `loadtest` template is used to set up the mycelo network).
 
-- Generate `bootstrap_balances.json` using the network's genesis block.
+- [Generate `bootstrap_balances.json`](#how-to-generate-bootstrap_balancesjson) within the same folder, using the mycelo network's genesis block.
 - In the `cli-config.json`:
-  - Set `network` to match the `ChainID` in the genesis file (not the network ID).
+  - Set `network` parameter to match the `ChainID` in the genesis file (not the network ID).
   - Point `bootstrap_balances` to the generated `bootstrap_balances.json`.
 
 #### Example

examples/generate_balances/main.go

@@ -17,9 +17,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"log"
-	"net/http"
 	"os"
 
 	"github.com/celo-org/celo-blockchain/core"
@@ -34,26 +32,20 @@ type BootstrapBalance struct {
 
 func main() {
 	if len(os.Args) != 3 {
-		fmt.Println("Usage: generate_balances <genesisURL> <outputFile>")
+		fmt.Println("Usage: generate_balances <pathToGenesis> <outputFile>")
 		os.Exit(1)
 	}
-	genesisURL := os.Args[1]
+	genesisPath := os.Args[1]
 	bootstrapBalancesFile := os.Args[2]
-	// Get Genesis File
-	resp, err := http.Get(genesisURL)
-	if err != nil {
-		log.Fatalln(err)
-	}
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
+	data, err := os.ReadFile(genesisPath)
 	if err != nil {
 		log.Fatalln(err)
 	}
 	genesis := &core.Genesis{}
-	if err := json.Unmarshal(body, genesis); err != nil {
+	if err := json.Unmarshal(data, genesis); err != nil {
 		log.Fatalln(err)
 	}
-	log.Printf("%+v\n", genesis.Alloc)
+	log.Println("Loaded genesis file, writing allocation file")
 	// Write to file
 	balances := []*BootstrapBalance{}
 	for k, v := range genesis.Alloc {
@@ -76,7 +68,7 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	if err := ioutil.WriteFile(bootstrapBalancesFile, file, os.FileMode(0600)); err != nil {
+	if err := os.WriteFile(bootstrapBalancesFile, file, os.FileMode(0600)); err != nil {
 		log.Fatal(err)
 	}
 	log.Printf("Bootstrap file contains %d balances\n", len(balances))

rosetta-cli-conf/alfajores/cli-config.json

@@ -13,7 +13,6 @@
   "tip_delay": 300,
   "log_configuration": false,
   "compression_disabled": false,
-  "memory_limit_disabled": false,
   "construction": {
     "offline_url": "http://localhost:8080",
     "max_offline_connections": 4,
@@ -59,13 +58,13 @@
     "interesting_accounts": "",
     "reconciliation_disabled": false,
     "reconciliation_drain_disabled": false,
-    "inactive_discrepency_search_disabled": false,
     "balance_tracking_disabled": false,
     "coin_tracking_disabled": false,
     "status_port": 9090,
     "results_output_file": "",
-    "pruning_disabled": false,
     "initial_balance_fetch_disabled": false,
+    "historical_balance_disabled": false,
+    "inactive_discrepancy_search_disabled": false,
     "end_conditions": {
       "reconciliation_coverage": {
         "coverage": 0.95,

rosetta-cli-conf/baklava/cli-config.json

@@ -37,7 +37,9 @@
     "inactive_discrepancy_search_disabled": false,
     "end_conditions": {
       "reconciliation_coverage": {
-        "coverage": 0.95
+        "coverage": 0.95,
+        "from_tip": true,
+        "tip": true
       }
     }
   }

rosetta-cli-conf/mainnet/cli-config.json

@@ -1,44 +1,46 @@
 {
-    "network": {
-      "blockchain": "celo",
-      "network": "42220"
-    },
-    "online_url": "http://localhost:8080",
-    "data_directory": "",
-    "http_timeout": 300,
-    "max_retries": 5,
-    "retry_elapsed_time": 0,
-    "max_online_connections": 120,
-    "max_sync_concurrency": 1,
-    "tip_delay": 300,
-    "log_configuration": false,
-    "compression_disabled": false,
-    "construction": null,
-    "data": {
-      "active_reconciliation_concurrency": 16,
-      "inactive_reconciliation_concurrency": 4,
-      "inactive_reconciliation_frequency": 250,
-      "log_blocks": false,
-      "log_transactions": false,
-      "log_balance_changes": false,
-      "log_reconciliations": false,
-      "ignore_reconciliation_error": false,
-      "exempt_accounts": "",
-      "bootstrap_balances": "bootstrap_balances.json",
-      "interesting_accounts": "",
-      "reconciliation_disabled": false,
-      "reconciliation_drain_disabled": false,
-      "inactive_discrepency_search_disabled": false,
-      "balance_tracking_disabled": false,
-      "coin_tracking_disabled": false,
-      "status_port": 9090,
-      "results_output_file": "",
-      "initial_balance_fetch_disabled": false,
-      "historical_balance_disabled": false,
-      "end_conditions": {
-        "reconciliation_coverage": {
-          "coverage": 0.95
-        }
+  "network": {
+    "blockchain": "celo",
+    "network": "42220"
+  },
+  "online_url": "http://localhost:8080",
+  "data_directory": "",
+  "http_timeout": 300,
+  "max_retries": 5,
+  "retry_elapsed_time": 0,
+  "max_online_connections": 120,
+  "max_sync_concurrency": 1,
+  "tip_delay": 300,
+  "log_configuration": false,
+  "compression_disabled": false,
+  "construction": null,
+  "data": {
+    "active_reconciliation_concurrency": 16,
+    "inactive_reconciliation_concurrency": 4,
+    "inactive_reconciliation_frequency": 250,
+    "log_blocks": false,
+    "log_transactions": false,
+    "log_balance_changes": false,
+    "log_reconciliations": false,
+    "ignore_reconciliation_error": false,
+    "exempt_accounts": "",
+    "bootstrap_balances": "bootstrap_balances.json",
+    "interesting_accounts": "",
+    "reconciliation_disabled": false,
+    "reconciliation_drain_disabled": false,
+    "inactive_discrepency_search_disabled": false,
+    "balance_tracking_disabled": false,
+    "coin_tracking_disabled": false,
+    "status_port": 9090,
+    "results_output_file": "",
+    "initial_balance_fetch_disabled": false,
+    "historical_balance_disabled": false,
+    "end_conditions": {
+      "reconciliation_coverage": {
+        "coverage": 0.95,
+        "from_tip": true,
+        "tip": true
       }
     }
   }
+}

rosetta-cli-conf/mycelo/cli-config.json

@@ -0,0 +1,45 @@
+{
+  "network": {
+    "blockchain": "celo",
+    "network": "9099000"
+  },
+  "online_url": "http://localhost:8080",
+  "data_directory": "",
+  "http_timeout": 300,
+  "max_retries": 5,
+  "retry_elapsed_time": 0,
+  "max_online_connections": 120,
+  "max_sync_concurrency": 1,
+  "tip_delay": 300,
+  "log_configuration": false,
+  "compression_disabled": false,
+  "construction": null,
+  "data": {
+    "active_reconciliation_concurrency": 16,
+    "inactive_reconciliation_concurrency": 4,
+    "inactive_reconciliation_frequency": 250,
+    "log_blocks": false,
+    "log_transactions": false,
+    "log_balance_changes": false,
+    "log_reconciliations": false,
+    "ignore_reconciliation_error": false,
+    "bootstrap_balances": "bootstrap_balances.json",
+    "interesting_accounts": "",
+    "reconciliation_disabled": false,
+    "reconciliation_drain_disabled": false,
+    "balance_tracking_disabled": false,
+    "coin_tracking_disabled": false,
+    "status_port": 9090,
+    "results_output_file": "",
+    "initial_balance_fetch_disabled": false,
+    "historical_balance_disabled": false,
+    "inactive_discrepancy_search_disabled": false,
+    "end_conditions": {
+      "reconciliation_coverage": {
+        "coverage": 0.95,
+        "from_tip": true,
+        "tip": true
+      }
+    }
+  }
+}