Retesteth Tool

README.md

@@ -69,6 +69,8 @@ Note: Do not modify this section! It is auto-generated by `cobra` using `make ge
 
 - [polycli parseethwallet](doc/polycli_parseethwallet.md) - Extract the private key from an eth wallet.
 
+- [polycli retest](doc/polycli_retest.md) - Convert the standard ETH test fillers into something to be replayed against an RPC
+
 - [polycli rpcfuzz](doc/polycli_rpcfuzz.md) - Continually run a variety of RPC calls and fuzzers.
 
 - [polycli signer](doc/polycli_signer.md) - Utilities for security signing transactions

cmd/enr/enr.go

@@ -2,8 +2,12 @@ package enr
 
 import (
 	_ "embed"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"github.com/ethereum/go-ethereum/core/forkid"
+	"github.com/ethereum/go-ethereum/p2p/enr"
+	"github.com/ethereum/go-ethereum/rlp"
 	"io"
 	"os"
 	"strings"
@@ -40,27 +44,36 @@ var ENRCmd = &cobra.Command{
 				continue
 			}
 			isENR := false
+			genericNode := make(map[string]any, 0)
+
 			if strings.HasPrefix(l, "enr:") {
 				isENR = true
 				node, err = enode.Parse(enode.V4ID{}, l)
 				if err != nil {
 					log.Error().Err(err).Str("line", l).Msg("Unable to parse enr record")
 					continue
 				}
+				var eth struct {
+					ForkID forkid.ID
+					Tail   []rlp.RawValue `rlp:"tail"`
+				}
+				_ = node.Load(enr.WithEntry("eth", &eth))
+				genericNode["forkid"] = ForkIDToString(eth.ForkID)
 			} else {
 				node, err = enode.ParseV4(l)
 				if err != nil {
 					log.Error().Err(err).Str("line", l).Msg("Unable to parse node record")
 					continue
 				}
 			}
-			genericNode := make(map[string]string, 0)
+
 			if isENR {
 				genericNode["enr"] = node.String()
 			}
 			genericNode["enode"] = node.URLv4()
 			genericNode["id"] = node.ID().String()
 			genericNode["ip"] = node.IP().String()
+			genericNode["seq"] = node.Seq()
 			genericNode["tcp"] = fmt.Sprintf("%d", node.TCP())
 			genericNode["udp"] = fmt.Sprintf("%d", node.UDP())
 			jsonOut, err := json.Marshal(genericNode)
@@ -77,6 +90,17 @@ var ENRCmd = &cobra.Command{
 	},
 }
 
+func ForkIDToString(id forkid.ID) string {
+	// Convert the Hash field to a hex string
+	hashHex := hex.EncodeToString(id.Hash[:])
+
+	// Convert the Next field to a hex string
+	nextHex := fmt.Sprintf("%016x", id.Next)
+
+	// Concatenate the two hex strings
+	return hashHex + nextHex
+}
+
 func init() {
 	flagSet := ENRCmd.PersistentFlags()
 	inputFileName = flagSet.String("file", "", "Provide a file that's holding ENRs")

cmd/hash/hash.go

@@ -1,6 +1,8 @@
 package hash
 
 import (
+	"bytes"
+	"encoding/binary"
 	"encoding/hex"
 	"fmt"
 	"hash"
@@ -22,6 +24,10 @@ import (
 	"golang.org/x/crypto/md4"       //nolint:staticcheck
 	"golang.org/x/crypto/ripemd160" //nolint:staticcheck
 	"golang.org/x/crypto/sha3"
+
+	"github.com/gateway-fm/vectorized-poseidon-gold/src/vectorizedposeidongold"
+	"github.com/iden3/go-iden3-crypto/poseidon"
+	"github.com/rs/zerolog/log"
 )
 
 var (
@@ -48,6 +54,8 @@ var (
 		"blake2b_512",
 		"keccak256",
 		"keccak512",
+		"poseidon",
+		"poseidongold",
 	}
 	inputFileName *string
 )
@@ -134,6 +142,11 @@ func getHash(name string) (hash.Hash, error) {
 		return sha3.NewLegacyKeccak256(), nil
 	case "keccak512":
 		return sha3.NewLegacyKeccak512(), nil
+	case "poseidon":
+		return poseidon.New(16)
+	case "poseidongold":
+		pg := new(poseidongoldWrapper)
+		return pg, nil
 	}
 	var h hash.Hash
 	return h, fmt.Errorf("unable to create a hash function for %s", name)
@@ -156,3 +169,95 @@ func getInputData(cmd *cobra.Command, args []string) ([]byte, error) {
 
 	return io.ReadAll(os.Stdin)
 }
+
+type poseidongoldWrapper struct {
+	b *bytes.Buffer
+}
+
+func (p *poseidongoldWrapper) Write(toWrite []byte) (n int, err error) {
+	if p.b == nil {
+		buf := make([]byte, 0)
+		p.b = bytes.NewBuffer(buf)
+	}
+	n, err = p.b.Write(toWrite)
+	if err != nil {
+		log.Error().Err(err).Msg("Unable to write to poseidon buffer")
+		return
+	}
+	return
+}
+
+func (p *poseidongoldWrapper) Sum(b []byte) []byte {
+	cap := [4]uint64{}
+	buf := make([]byte, 64)
+
+	var err error
+	for {
+		n, _ := p.b.Read(buf)
+		log.Info().Bytes("current-buffer", buf).Msg("summing")
+		if n > 64 {
+			panic("What?? that shouldn't have happened")
+		}
+		hashInput, _ := bytesToUints(buf)
+		cap, err = vectorizedposeidongold.Hash(hashInput, cap)
+		if err != nil {
+			log.Error().Err(err).Msg("Unable to hash input")
+			// Exit since we don't want to return a known bad hash value here
+			os.Exit(1)
+		}
+		if n < 64 {
+			log.Info().Int("n", n).Msg("done")
+			break
+		}
+
+	}
+	return Uint64ArrayToBytes(cap)
+}
+func (p *poseidongoldWrapper) Reset() {
+	p.b.Reset()
+}
+func (p *poseidongoldWrapper) Size() int {
+	return 32
+}
+func (p *poseidongoldWrapper) BlockSize() int {
+	return 64
+}
+
+func bytesToUints(buf []byte) ([8]uint64, error) {
+	if len(buf) > 64 {
+		return [8]uint64{}, fmt.Errorf("the underlying data is too large. Expected less than 64 bytes but got %d", len(buf))
+	}
+
+	var result [8]uint64
+	for i := 0; i < len(buf); i += 8 {
+		var val uint64
+		for j := 0; j < 8 && i+j < len(buf); j++ {
+			val |= uint64(buf[i+j]) << (8 * j)
+		}
+		result[i/8] = val
+	}
+
+	return result, nil
+}
+
+func Uint64ArrayToBytes(arr [4]uint64) []byte {
+	buf := new(bytes.Buffer)
+	for k := range arr {
+		// this serialization seems a little weird, but it matches the values from erigion as far as I can tell.
+		// echo -n "0000" | xxd -r -p | ./out/polycli hash poseidongold
+		// This returns
+		// c71603f33a1144ca7953db0ab48808f4c4055e3364a246c33c18a9786cb0b359
+		// which matches
+		// https://github.com/0xPolygonHermez/cdk-erigon/blob/92e7d22534bc103203a4a699b8de60a4a5df4c3e/smt/pkg/utils/utils.go#L24
+		//
+		// This doesn't quite match the order from the tests in hermez and plonky
+		// https://github.com/0xPolygonHermez/goldilocks/blob/a0a516ac38145049a527e6c822e11c8df108d6d1/tests/tests.cpp#L1515-L1518
+		// https://github.com/0xPolygonZero/plonky2/blob/349beae1431ecffc1bf8c044d6c00e2bf194b74a/plonky2/src/hash/poseidon_goldilocks.rs#L468
+		err := binary.Write(buf, binary.BigEndian, arr[4-k-1])
+		if err != nil {
+			// This error handling is mainly for demonstration, it should never occur for fixed size and types.
+			fmt.Println("binary.Write failed:", err)
+		}
+	}
+	return buf.Bytes()
+}

cmd/loadtest/app.go

@@ -79,6 +79,7 @@ type (
 		ContractCallPayable           *bool
 		InscriptionContent            *string
 		BlobFeeCap                    *uint64
+		StartNonce                    *uint64
 
 		// Computed
 		CurrentGasPrice     *big.Int
@@ -224,6 +225,7 @@ func initFlags() {
 	ltp.Seed = LoadtestCmd.PersistentFlags().Int64("seed", 123456, "A seed for generating random values and addresses")
 	ltp.ForceGasLimit = LoadtestCmd.PersistentFlags().Uint64("gas-limit", 0, "In environments where the gas limit can't be computed on the fly, we can specify it manually. This can also be used to avoid eth_estimateGas")
 	ltp.ForceGasPrice = LoadtestCmd.PersistentFlags().Uint64("gas-price", 0, "In environments where the gas price can't be determined automatically, we can specify it manually")
+	ltp.StartNonce = LoadtestCmd.PersistentFlags().Uint64("nonce", 0, "Use this flag to manually set the starting nonce")
 	ltp.ForcePriorityGasPrice = LoadtestCmd.PersistentFlags().Uint64("priority-gas-price", 0, "Specify Gas Tip Price in the case of EIP-1559")
 	ltp.ShouldProduceSummary = LoadtestCmd.PersistentFlags().Bool("summarize", false, "Should we produce an execution summary after the load test has finished. If you're running a large load test, this can take a long time")
 	ltp.BatchSize = LoadtestCmd.PersistentFlags().Uint64("batch-size", 999, "Number of batches to perform at a time for receipt fetching. Default is 999 requests at a time.")

cmd/loadtest/loadtest.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"math/big"
 	"math/rand"
+	"net/http"
 
 	"os"
 	"os/signal"
@@ -71,47 +72,47 @@ const (
 	loadTestModeStore
 	loadTestModeTransaction
 	loadTestModeUniswapV3
-	
+
 	codeQualitySeed       = "code code code code code code code code code code code quality"
 	codeQualityPrivateKey = "42b6e34dc21598a807dc19d7784c71b2a7a01f6480dc6f58258f78e539f1a1fa"
 )
 
 func characterToLoadTestMode(mode string) (loadTestMode, error) {
 	switch mode {
 	case "2", "erc20":
-    return loadTestModeERC20, nil
+		return loadTestModeERC20, nil
 	case "7", "erc721":
-			return loadTestModeERC721, nil
+		return loadTestModeERC721, nil
 	case "b", "blob":
-			return loadTestModeBlob, nil
+		return loadTestModeBlob, nil
 	case "c", "call":
-			return loadTestModeCall, nil
+		return loadTestModeCall, nil
 	case "cc", "contract-call":
-			return loadTestModeContractCall, nil
+		return loadTestModeContractCall, nil
 	case "d", "deploy":
-			return loadTestModeDeploy, nil
+		return loadTestModeDeploy, nil
 	case "f", "function":
-			return loadTestModeFunction, nil
+		return loadTestModeFunction, nil
 	case "i", "inscription":
 		return loadTestModeInscription, nil
 	case "inc", "increment":
-			return loadTestModeIncrement, nil
+		return loadTestModeIncrement, nil
 	case "pr", "random-precompile":
-			return loadTestModeRandomPrecompiledContract, nil
+		return loadTestModeRandomPrecompiledContract, nil
 	case "px", "specific-precompile":
-			return loadTestModeSpecificPrecompiledContract, nil
+		return loadTestModeSpecificPrecompiledContract, nil
 	case "r", "random":
-			return loadTestModeRandom, nil
+		return loadTestModeRandom, nil
 	case "R", "recall":
-			return loadTestModeRecall, nil
+		return loadTestModeRecall, nil
 	case "rpc":
-			return loadTestModeRPC, nil
+		return loadTestModeRPC, nil
 	case "s", "store":
-			return loadTestModeStore, nil
+		return loadTestModeStore, nil
 	case "t", "transaction":
-			return loadTestModeTransaction, nil
+		return loadTestModeTransaction, nil
 	case "v3", "uniswapv3":
-			return loadTestModeUniswapV3, nil
+		return loadTestModeUniswapV3, nil
 	default:
 		return 0, fmt.Errorf("unrecognized load test mode: %s", mode)
 	}
@@ -324,6 +325,9 @@ func initializeLoadTestParams(ctx context.Context, c *ethclient.Client) error {
 }
 
 func initNonce(ctx context.Context, c *ethclient.Client, rpc *ethrpc.Client) error {
+	currentNonceMutex.Lock()
+	defer currentNonceMutex.Unlock()
+
 	var err error
 	startBlockNumber, err = c.BlockNumber(ctx)
 	if err != nil {
@@ -333,13 +337,18 @@ func initNonce(ctx context.Context, c *ethclient.Client, rpc *ethrpc.Client) err
 
 	// Get pending nonce to be prevent nonce collision (if tx from same sender is already present)
 	currentNonce, err = c.PendingNonceAt(ctx, *inputLoadTestParams.FromETHAddress)
-	startNonce = currentNonce
-
 	if err != nil {
 		log.Error().Err(err).Msg("Unable to get account nonce")
 		return err
 	}
 
+	if inputLoadTestParams.StartNonce != nil && *inputLoadTestParams.StartNonce > 0 {
+		currentNonce = *inputLoadTestParams.StartNonce
+	}
+
+	log.Info().Uint64("startNonce", startNonce).Msg("setting the starting nonce")
+
+	startNonce = currentNonce
 	return nil
 }
 
@@ -396,8 +405,21 @@ func runLoadTest(ctx context.Context) error {
 		overallTimer = new(time.Timer)
 	}
 
-	// Dial the Ethereum RPC server.
-	rpc, err := ethrpc.DialContext(ctx, *inputLoadTestParams.RPCUrl)
+	// connLimit is the value we'll use to configure the connection limit within the http transport
+	connLimit := 2 * int(*inputLoadTestParams.Concurrency)
+	// Most of these transport options are defaults. We might want to make this configurable from the CLI at some point.
+	// The goal here is to avoid opening a ton of connections that go idle then get closed and eventually exhausting
+	// client-side connections.
+	transport := &http.Transport{
+		MaxIdleConns:        connLimit,
+		MaxIdleConnsPerHost: connLimit,
+		MaxConnsPerHost:     connLimit,
+	}
+	goHttpClient := &http.Client{
+		Transport: transport,
+	}
+	rpcOption := ethrpc.WithHTTPClient(goHttpClient)
+	rpc, err := ethrpc.DialOptions(ctx, *inputLoadTestParams.RPCUrl, rpcOption)
 	if err != nil {
 		log.Error().Err(err).Msg("Unable to dial rpc")
 		return err
@@ -721,6 +743,13 @@ func mainLoop(ctx context.Context, c *ethclient.Client, rpc *ethrpc.Client) erro
 					if strings.Contains(tErr.Error(), "nonce too low") && retryForNonce {
 						retryForNonce = false
 					}
+					if strings.Contains(tErr.Error(), "already known") && retryForNonce {
+						retryForNonce = false
+					}
+					if strings.Contains(tErr.Error(), "could not replace existing") && retryForNonce {
+						retryForNonce = false
+					}
+
 				}
 
 				log.Trace().Uint64("nonce", myNonceValue).Int64("routine", i).Str("mode", localMode.String()).Int64("request", j).Msg("Request")