Merge branch 'main' into feat/fuzz-testing

Makefile

@@ -37,7 +37,7 @@ build: $(BUILD_DIR) ## Build go binary.
 .PHONY: install
 install: build ## Install the go binary.
 	$(RM) $(INSTALL_DIR)/$(BIN_NAME)
-	cp $(BUILD_DIR)/$(BIN_NAME) $(INSTALL_DIR)
+	cp $(BUILD_DIR)/$(BIN_NAME) $(INSTALL_DIR)/
 
 .PHONY: cross
 cross: $(BUILD_DIR) ## Cross-compile go binaries using CGO.
@@ -119,4 +119,4 @@ geth-loadtest: build ## Fund test account with 5k ETH and run loadtest against a
 
 .PHONY: avail-loadtest
 avail-loadtest: build ## Run loadtest against an Avail chain.
-	$(BUILD_DIR)/$(BIN_NAME) loadtest --verbosity 700 --chain-id 1256 --concurrency 1 --requests 1000 --rate-limit 5 --mode t --data-avail http://127.0.0.1:$(PORT)
+	$(BUILD_DIR)/$(BIN_NAME) loadtest --verbosity 700 --chain-id 1256 --concurrency 1 --requests 1000 --rate-limit 5 --mode t --data-avail http://127.0.0.1:$(PORT)

README.md

@@ -8,6 +8,9 @@ commonly needed tools and provide interfaces and formats.
 Requirements:
 
 - [Go](https://go.dev/)
+- make
+- jq
+- bc
 
 To install, clone this repo and run:
 

cmd/forge/forge.go

@@ -47,7 +47,6 @@ import (
 	"github.com/maticnetwork/polygon-cli/rpctypes"
 	"golang.org/x/exp/slices"
 
-	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 )
@@ -133,10 +132,6 @@ Here is an example usage:
 		}
 		inputForge.GenesisData = genesisData
 
-		zerolog.SetGlobalLevel(zerolog.TraceLevel)
-		log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
-		log.Debug().Msg("Starting logger in console mode")
-
 		return nil
 	},
 }
@@ -145,17 +140,17 @@ func init() {
 	ForgeCmd.PersistentFlags().StringVarP(&inputForge.Client, "client", "c", "edge", "Specify which blockchain client should be use to forge the data")
 	ForgeCmd.PersistentFlags().StringVarP(&inputForge.DataDir, "data-dir", "d", "./forged-data", "Specify a folder to be used to store the chain data")
 	ForgeCmd.PersistentFlags().StringVarP(&inputForge.GenesisFile, "genesis", "g", "genesis.json", "Specify a file to be used for genesis configuration")
-	ForgeCmd.PersistentFlags().StringVarP(&inputForge.Verifier, "verifier", "v", "dummy", "Specify a consensus engine to use for forging")
+	ForgeCmd.PersistentFlags().StringVarP(&inputForge.Verifier, "verifier", "V", "dummy", "Specify a consensus engine to use for forging")
 	ForgeCmd.PersistentFlags().StringVarP(&inputForge.Mode, "mode", "m", "json", "The forge mode indicates how we should get the transactions for our blocks [json, proto]")
 	ForgeCmd.PersistentFlags().Uint64VarP(&inputForge.Count, "count", "C", 100, "The number of blocks to try to forge")
 	ForgeCmd.PersistentFlags().StringVarP(&inputForge.BlocksFile, "blocks", "b", "", "A file of encoded blocks; the format of this file should match the mode")
 	ForgeCmd.PersistentFlags().StringVarP(&inputForge.BaseBlockReward, "base-block-reward", "B", "2_000_000_000_000_000_000", "The amount rewarded for mining blocks")
 	ForgeCmd.PersistentFlags().StringVarP(&inputForge.ReceiptsFile, "receipts", "r", "", "A file of encoded receipts; the format of this file should match the mode")
 	ForgeCmd.PersistentFlags().BoolVarP(&inputForge.IncludeTxFees, "tx-fees", "t", false, "if the transaction fees should be included when computing block rewards")
 	ForgeCmd.PersistentFlags().BoolVarP(&inputForge.ShouldReadFirstBlock, "read-first-block", "R", false, "whether to read the first block, leave false if first block is genesis")
-	ForgeCmd.PersistentFlags().BoolVarP(&inputForge.ShouldVerifyBlocks, "verify-blocks", "V", true, "whether to verify blocks, set false if forging nonconsecutive blocks")
-	ForgeCmd.PersistentFlags().BoolVarP(&inputForge.ShouldRewriteTxNonces, "rewrite-tx-nonces", "", false, "whether to rewrite transaction nonces, set true if forging nonconsecutive blocks")
-	ForgeCmd.PersistentFlags().BoolVarP(&inputForge.HasConsecutiveBlocks, "consecutive-blocks", "", true, "whether the blocks file has consecutive blocks")
+	ForgeCmd.PersistentFlags().BoolVar(&inputForge.ShouldVerifyBlocks, "verify-blocks", true, "whether to verify blocks, set false if forging nonconsecutive blocks")
+	ForgeCmd.PersistentFlags().BoolVar(&inputForge.ShouldRewriteTxNonces, "rewrite-tx-nonces", false, "whether to rewrite transaction nonces, set true if forging nonconsecutive blocks")
+	ForgeCmd.PersistentFlags().BoolVar(&inputForge.HasConsecutiveBlocks, "consecutive-blocks", true, "whether the blocks file has consecutive blocks")
 	ForgeCmd.PersistentFlags().BoolVarP(&inputForge.ShouldProcessBlocks, "process-blocks", "p", true, "whether the transactions in blocks should be processed applied to the state")
 
 	if err := cobra.MarkFlagRequired(ForgeCmd.PersistentFlags(), "blocks"); err != nil {
@@ -345,7 +340,7 @@ func readAllBlocksToChain(bh *edgeBlockchainHandle, blockReader BlockReader, rec
 		if inputForge.ShouldRewriteTxNonces {
 			for nonce, tx := range edgeBlock.Transactions {
 				tx.Nonce = uint64(nonce)
-				log.Logger.Debug().Int64("old nonce", int64(tx.Nonce)).Int64("new nonce", int64(nonce)).Str("tx hash", tx.Hash.String()).Msg("Rewrote tx nonce")
+				log.Debug().Int64("old nonce", int64(tx.Nonce)).Int64("new nonce", int64(nonce)).Str("tx hash", tx.Hash.String()).Msg("Rewrote tx nonce")
 			}
 		}
 

cmd/fork/fork.go

@@ -8,7 +8,6 @@ import (
 	"github.com/ethereum/go-ethereum/consensus/clique"
 	"github.com/ethereum/go-ethereum/core/types"
 	ethcrypto "github.com/ethereum/go-ethereum/crypto"
-	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"os"
@@ -175,10 +174,3 @@ func ecrecover(block *types.Block) ([]byte, error) {
 
 	return signer, nil
 }
-
-func init() {
-	// flagSet := ForkCmd.PersistentFlags()
-	zerolog.SetGlobalLevel(zerolog.TraceLevel)
-	log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
-
-}

cmd/loadtest/loadtest.go

@@ -162,7 +162,6 @@ var LoadtestCmd = &cobra.Command{
 		return nil
 	},
 	Args: func(cmd *cobra.Command, args []string) error {
-		setLogLevel(inputLoadTestParams)
 		zerolog.DurationFieldUnit = time.Second
 		zerolog.DurationFieldInteger = true
 
@@ -189,31 +188,6 @@ var LoadtestCmd = &cobra.Command{
 	},
 }
 
-func setLogLevel(ltp loadTestParams) {
-	verbosity := *ltp.Verbosity
-	if verbosity < 100 {
-		zerolog.SetGlobalLevel(zerolog.PanicLevel)
-	} else if verbosity < 200 {
-		zerolog.SetGlobalLevel(zerolog.FatalLevel)
-	} else if verbosity < 300 {
-		zerolog.SetGlobalLevel(zerolog.ErrorLevel)
-	} else if verbosity < 400 {
-		zerolog.SetGlobalLevel(zerolog.WarnLevel)
-	} else if verbosity < 500 {
-		zerolog.SetGlobalLevel(zerolog.InfoLevel)
-	} else if verbosity < 600 {
-		zerolog.SetGlobalLevel(zerolog.DebugLevel)
-	} else {
-		zerolog.SetGlobalLevel(zerolog.TraceLevel)
-	}
-	if *ltp.PrettyLogs {
-		log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
-		log.Debug().Msg("Starting logger in console mode")
-	} else {
-		log.Debug().Msg("Starting logger in JSON mode")
-	}
-}
-
 type (
 	blockSummary struct {
 		Block     *rpctypes.RawBlockResponse
@@ -237,8 +211,6 @@ type (
 		Concurrency                         *int64
 		BatchSize                           *uint64
 		TimeLimit                           *int64
-		Verbosity                           *int64
-		PrettyLogs                          *bool
 		ToRandom                            *bool
 		URL                                 *url.URL
 		ChainID                             *uint64
@@ -293,10 +265,8 @@ func init() {
 	ltp.Concurrency = LoadtestCmd.PersistentFlags().Int64P("concurrency", "c", 1, "Number of multiple requests to perform at a time. Default is one request at a time.")
 	ltp.TimeLimit = LoadtestCmd.PersistentFlags().Int64P("time-limit", "t", -1, "Maximum number of seconds to spend for benchmarking. Use this to benchmark within a fixed total amount of time. Per default there is no timelimit.")
 	// https://logging.apache.org/log4j/2.x/manual/customloglevels.html
-	ltp.Verbosity = LoadtestCmd.PersistentFlags().Int64P("verbosity", "v", 200, "0 - Silent\n100 Fatals\n200 Errors\n300 Warnings\n400 INFO\n500 Debug\n600 Trace")
 
 	// extended parameters
-	ltp.PrettyLogs = LoadtestCmd.PersistentFlags().Bool("pretty-logs", true, "Should we log in pretty format or JSON")
 	ltp.PrivateKey = LoadtestCmd.PersistentFlags().String("private-key", codeQualityPrivateKey, "The hex encoded private key that we'll use to sending transactions")
 	ltp.ChainID = LoadtestCmd.PersistentFlags().Uint64("chain-id", 1256, "The chain id for the transactions that we're going to send")
 	ltp.ToAddress = LoadtestCmd.PersistentFlags().String("to-address", "0xDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF", "The address that we're going to send to")
@@ -577,32 +547,35 @@ func getTxPoolSize(rpc *ethrpc.Client) (uint64, error) {
 	return pendingCount + queuedCount, nil
 }
 
-func updateRateLimit(rl *rate.Limiter, rpc *ethrpc.Client, steadyStateQueueSize uint64, rateLimitIncrement uint64, cycleDuration time.Duration, backoff float64) {
+func updateRateLimit(ctx context.Context, rl *rate.Limiter, rpc *ethrpc.Client, steadyStateQueueSize uint64, rateLimitIncrement uint64, cycleDuration time.Duration, backoff float64) {
 	ticker := time.NewTicker(cycleDuration)
 	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			txPoolSize, err := getTxPoolSize(rpc)
+			if err != nil {
+				log.Error().Err(err).Msg("Error getting txpool size")
+				return
+			}
 
-	for range ticker.C {
-		txPoolSize, err := getTxPoolSize(rpc)
-		if err != nil {
-			log.Error().Err(err).Msg("Error getting txpool size")
+			if txPoolSize < steadyStateQueueSize {
+				// additively increment requests per second if txpool less than queue steady state
+				newRateLimit := rate.Limit(float64(rl.Limit()) + float64(rateLimitIncrement))
+				rl.SetLimit(newRateLimit)
+				log.Info().Float64("New Rate Limit (RPS)", float64(rl.Limit())).Uint64("Current Tx Pool Size", txPoolSize).Uint64("Steady State Tx Pool Size", steadyStateQueueSize).Msg("Increased rate limit")
+			} else if txPoolSize > steadyStateQueueSize {
+				// halve rate limit requests per second if txpool greater than queue steady state
+				rl.SetLimit(rl.Limit() / rate.Limit(backoff))
+				log.Info().Float64("New Rate Limit (RPS)", float64(rl.Limit())).Uint64("Current Tx Pool Size", txPoolSize).Uint64("Steady State Tx Pool Size", steadyStateQueueSize).Msg("Backed off rate limit")
+			}
+		case <-ctx.Done():
 			return
 		}
-
-		if txPoolSize < steadyStateQueueSize {
-			// additively increment requests per second if txpool less than queue steady state
-			newRateLimit := rate.Limit(float64(rl.Limit()) + float64(rateLimitIncrement))
-			rl.SetLimit(newRateLimit)
-			log.Info().Float64("New Rate Limit (RPS)", float64(rl.Limit())).Uint64("Current Tx Pool Size", txPoolSize).Uint64("Steady State Tx Pool Size", steadyStateQueueSize).Msg("Increased rate limit")
-		} else if txPoolSize > steadyStateQueueSize {
-			// halve rate limit requests per second if txpool greater than queue steady state
-			rl.SetLimit(rl.Limit() / rate.Limit(backoff))
-			log.Info().Float64("New Rate Limit (RPS)", float64(rl.Limit())).Uint64("Current Tx Pool Size", txPoolSize).Uint64("Steady State Tx Pool Size", steadyStateQueueSize).Msg("Backed off rate limit")
-		}
 	}
 }
 
 func mainLoop(ctx context.Context, c *ethclient.Client, rpc *ethrpc.Client) error {
-
 	ltp := inputLoadTestParams
 	log.Trace().Interface("Input Params", ltp).Msg("Params")
 
@@ -619,8 +592,10 @@ func mainLoop(ctx context.Context, c *ethclient.Client, rpc *ethrpc.Client) erro
 	if *ltp.RateLimit <= 0.0 {
 		rl = nil
 	}
+	rateLimitCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	if *ltp.AdaptiveRateLimit && rl != nil {
-		go updateRateLimit(rl, rpc, steadyStateTxPoolSize, adaptiveRateLimitIncrement, time.Duration(*ltp.AdaptiveCycleDuration)*time.Second, *ltp.AdaptiveBackoffFactor)
+		go updateRateLimit(rateLimitCtx, rl, rpc, steadyStateTxPoolSize, adaptiveRateLimitIncrement, time.Duration(*ltp.AdaptiveCycleDuration)*time.Second, *ltp.AdaptiveBackoffFactor)
 	}
 
 	tops, err := bind.NewKeyedTransactorWithChainID(privateKey, chainID)
@@ -872,10 +847,10 @@ func mainLoop(ctx context.Context, c *ethclient.Client, rpc *ethrpc.Client) erro
 			}
 			wg.Done()
 		}(i)
-
 	}
 	log.Trace().Msg("Finished starting go routines. Waiting..")
 	wg.Wait()
+	cancel()
 	log.Debug().Uint64("currentNonce", currentNonce).Msg("Finished main loadtest loop")
 	log.Debug().Msg("Waiting for transactions to actually be mined")
 	finalBlockNumber, err := waitForFinalBlock(ctx, c, rpc, startBlockNumber, startNonce, currentNonce)

cmd/monitor/monitor.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"math/big"
 	"net/url"
-	"os"
 	"sort"
 	"sync"
 	"time"
@@ -34,18 +33,15 @@ import (
 	"github.com/gizak/termui/v3/widgets"
 	"github.com/maticnetwork/polygon-cli/metrics"
 	"github.com/maticnetwork/polygon-cli/rpctypes"
-	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 )
 
 var (
 	batchSize   uint64
 	windowSize  int
-	verbosity   int64
 	intervalStr string
 	interval    time.Duration
-	logFile     string
 
 	one  = big.NewInt(1)
 	zero = big.NewInt(0)
@@ -132,7 +128,7 @@ var MonitorCmd = &cobra.Command{
 			return err
 		}
 
-		return setMonitorLogLevel(verbosity)
+		return nil
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
 		ctx := cmd.Context()
@@ -268,9 +264,7 @@ func (ms *monitorStatus) getBlockRange(ctx context.Context, from, to *big.Int, c
 func init() {
 	MonitorCmd.PersistentFlags().Uint64VarP(&batchSize, "batch-size", "b", 25, "Number of requests per batch")
 	MonitorCmd.PersistentFlags().IntVarP(&windowSize, "window-size", "w", 25, "Number of blocks visible in the window")
-	MonitorCmd.PersistentFlags().Int64VarP(&verbosity, "verbosity", "v", 200, "0 - Silent\n100 Fatal\n200 Error\n300 Warning\n400 Info\n500 Debug\n600 Trace")
 	MonitorCmd.PersistentFlags().StringVarP(&intervalStr, "interval", "i", "5s", "Amount of time between batch block rpc calls")
-	MonitorCmd.PersistentFlags().StringVarP(&logFile, "log-file", "l", "", "Write logs to a file (default stderr)")
 }
 
 func renderMonitorUI(ms *monitorStatus) error {
@@ -586,34 +580,3 @@ func max(nums ...int) int {
 	}
 	return m
 }
-
-// setMonitorLogLevel sets the log level based on the flags. If the log file flag
-// is set, then output will be written to the file instead of stdout. Use
-// `tail -f <log file>` to see the log output in real time.
-func setMonitorLogLevel(verbosity int64) error {
-	if len(logFile) > 0 {
-		file, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0664)
-		if err != nil {
-			return err
-		}
-		log.Logger = zerolog.New(file).With().Logger()
-	}
-
-	if verbosity < 100 {
-		zerolog.SetGlobalLevel(zerolog.PanicLevel)
-	} else if verbosity < 200 {
-		zerolog.SetGlobalLevel(zerolog.FatalLevel)
-	} else if verbosity < 300 {
-		zerolog.SetGlobalLevel(zerolog.ErrorLevel)
-	} else if verbosity < 400 {
-		zerolog.SetGlobalLevel(zerolog.WarnLevel)
-	} else if verbosity < 500 {
-		zerolog.SetGlobalLevel(zerolog.InfoLevel)
-	} else if verbosity < 600 {
-		zerolog.SetGlobalLevel(zerolog.DebugLevel)
-	} else {
-		zerolog.SetGlobalLevel(zerolog.TraceLevel)
-	}
-
-	return nil
-}

cmd/p2p/p2p.go

@@ -1,47 +1,20 @@
 package p2p
 
 import (
-	"github.com/rs/zerolog"
 	"github.com/spf13/cobra"
 
 	"github.com/maticnetwork/polygon-cli/cmd/p2p/crawl"
 	"github.com/maticnetwork/polygon-cli/cmd/p2p/ping"
 	"github.com/maticnetwork/polygon-cli/cmd/p2p/sensor"
 )
 
-var verbosity int
-
 var P2pCmd = &cobra.Command{
 	Use:   "p2p",
 	Short: "Commands related to devp2p",
-	PersistentPreRun: func(cmd *cobra.Command, args []string) {
-		setMonitorLogLevel(verbosity)
-	},
 }
 
 func init() {
-	P2pCmd.PersistentFlags().IntVarP(&verbosity, "verbosity", "v", 400, "0 - Silent\n100 Fatal\n200 Error\n300 Warning\n400 Info\n500 Debug\n600 Trace")
-
 	P2pCmd.AddCommand(sensor.SensorCmd)
 	P2pCmd.AddCommand(crawl.CrawlCmd)
 	P2pCmd.AddCommand(ping.PingCmd)
 }
-
-// setMonitorLogLevel sets the log level based on the flags.
-func setMonitorLogLevel(verbosity int) {
-	if verbosity < 100 {
-		zerolog.SetGlobalLevel(zerolog.PanicLevel)
-	} else if verbosity < 200 {
-		zerolog.SetGlobalLevel(zerolog.FatalLevel)
-	} else if verbosity < 300 {
-		zerolog.SetGlobalLevel(zerolog.ErrorLevel)
-	} else if verbosity < 400 {
-		zerolog.SetGlobalLevel(zerolog.WarnLevel)
-	} else if verbosity < 500 {
-		zerolog.SetGlobalLevel(zerolog.InfoLevel)
-	} else if verbosity < 600 {
-		zerolog.SetGlobalLevel(zerolog.DebugLevel)
-	} else {
-		zerolog.SetGlobalLevel(zerolog.TraceLevel)
-	}
-}

cmd/p2p/ping/ping.go

@@ -19,7 +19,6 @@ type (
 		OutputFile string
 		NodesFile  string
 		Listen     bool
-		Verbosity  int
 	}
 	pingNodeJSON struct {
 		Record *enode.Node `json:"record"`