Merge branch 'zkevm' into discard-buggy-transactions

.github/workflows/ci_zkevm.yml

@@ -21,9 +21,9 @@ jobs:
   tests:
     strategy:
       matrix:
-        os: [ ubuntu-20.04, macos-13 ] # list of os: https://github.com/actions/virtual-environments
+        os: [ ubuntu-20.04, macos-13-xlarge ] # list of os: https://github.com/actions/virtual-environments
     runs-on: ${{ matrix.os }}
-    timeout-minutes: ${{ matrix.os == 'macos-14' && 40 || 30 }}
+    timeout-minutes: ${{ matrix.os == 'macos-14-xlarge' && 40 || 30 }}
 
     steps:
       - uses: actions/checkout@v3

Makefile

@@ -149,7 +149,7 @@ db-tools:
 
 ## test:                              run unit tests with a 100s timeout
 test:
-	$(GOTEST) --timeout 200s
+	$(GOTEST) --timeout 10m
 
 test3:
 	$(GOTEST) --timeout 200s -tags $(BUILD_TAGS),erigon3

chain/chain_config.go

@@ -102,7 +102,7 @@ type Config struct {
 	ForkID11                *big.Int `json:"forkID11,omitempty"`
 	ForkID12BananaBlock     *big.Int `json:"forkID12BananaBlock,omitempty"`
 
-	SupportGasless bool `json:"supportGasless,omitempty"`
+	AllowFreeTransactions bool `json:"allowFreeTransactions,omitempty"`
 }
 
 func (c *Config) String() string {

cmd/cdk-erigon/main.go

@@ -18,8 +18,8 @@ import (
 	"github.com/ledgerwatch/erigon/params"
 	erigonapp "github.com/ledgerwatch/erigon/turbo/app"
 	erigoncli "github.com/ledgerwatch/erigon/turbo/cli"
-	"github.com/ledgerwatch/erigon/turbo/node"
 	"github.com/ledgerwatch/erigon/turbo/logging"
+	"github.com/ledgerwatch/erigon/turbo/node"
 )
 
 func main() {
@@ -109,13 +109,17 @@ func setFlagsFromConfigFile(ctx *cli.Context, filePath string) error {
 				for i, v := range sliceInterface {
 					s[i] = fmt.Sprintf("%v", v)
 				}
-				err := ctx.Set(key, strings.Join(s, ","))
-				if err != nil {
+				if err := ctx.Set(key, strings.Join(s, ",")); err != nil {
+					if deprecatedFlag, found := erigoncli.DeprecatedFlags[key]; found {
+						return fmt.Errorf("failed setting %s flag Flag is deprecated, use %s instead", key, deprecatedFlag)
+					}
 					return fmt.Errorf("failed setting %s flag with values=%s error=%s", key, s, err)
 				}
 			} else {
-				err := ctx.Set(key, fmt.Sprintf("%v", value))
-				if err != nil {
+				if err := ctx.Set(key, fmt.Sprintf("%v", value)); err != nil {
+					if deprecatedFlag, found := erigoncli.DeprecatedFlags[key]; found {
+						return fmt.Errorf("failed setting %s flag Flag is deprecated, use %s instead", key, deprecatedFlag)
+					}
 					return fmt.Errorf("failed setting %s flag with value=%v error=%s", key, value, err)
 				}
 			}

cmd/integration/commands/flags.go

@@ -31,6 +31,7 @@ var (
 	pruneTBefore, pruneCBefore              uint64
 	experiments                             []string
 	chain                                   string // Which chain to use (mainnet, rinkeby, goerli, etc.)
+	config                                  string
 
 	commitmentMode string
 	commitmentTrie string
@@ -49,7 +50,7 @@ func must(err error) {
 }
 
 func withConfig(cmd *cobra.Command) {
-	cmd.Flags().String("config", "", "yaml/toml config file location")
+	cmd.Flags().StringVar(&config, "config", "", "yaml/toml config file location")
 }
 
 func withMining(cmd *cobra.Command) {

cmd/integration/commands/stage_stages_zkevm.go

@@ -8,11 +8,8 @@ import (
 
 	common2 "github.com/gateway-fm/cdk-erigon-lib/common"
 	"github.com/gateway-fm/cdk-erigon-lib/kv"
-	"github.com/ledgerwatch/erigon/core"
-	"github.com/ledgerwatch/erigon/eth/ethconfig"
 	"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
 	smtdb "github.com/ledgerwatch/erigon/smt/pkg/db"
-	erigoncli "github.com/ledgerwatch/erigon/turbo/cli"
 	"github.com/ledgerwatch/erigon/zk/hermez_db"
 	"github.com/ledgerwatch/log/v3"
 	"github.com/spf13/cobra"
@@ -28,9 +25,6 @@ state_stages_zkevm --datadir=/datadirs/hermez-mainnet --unwind-batch-no=2 --chai
 	Example: "go run ./cmd/integration state_stages_zkevm --config=... --verbosity=3 --unwind-batch-no=100",
 	Run: func(cmd *cobra.Command, args []string) {
 		ctx, _ := common2.RootContext()
-		ethConfig := &ethconfig.Defaults
-		ethConfig.Genesis = core.GenesisBlockByChainName(chain)
-		erigoncli.ApplyFlagsForEthConfigCobra(cmd.Flags(), ethConfig)
 		db := openDB(dbCfg(kv.ChainDB, chaindata), true)
 		defer db.Close()
 

cmd/integration/commands/stages_zkevm.go

@@ -2,6 +2,12 @@ package commands
 
 import (
 	"context"
+	"encoding/json"
+	"math/big"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
 
 	"github.com/c2h5oh/datasize"
 	chain3 "github.com/gateway-fm/cdk-erigon-lib/chain"
@@ -10,11 +16,14 @@ import (
 	"github.com/gateway-fm/cdk-erigon-lib/kv/kvcfg"
 	"github.com/ledgerwatch/erigon/cmd/hack/tool/fromdb"
 	"github.com/ledgerwatch/erigon/cmd/sentry/sentry"
+	"github.com/ledgerwatch/erigon/cmd/utils"
 	"github.com/ledgerwatch/erigon/consensus"
 	"github.com/ledgerwatch/erigon/core"
+	"github.com/ledgerwatch/erigon/core/types"
 	"github.com/ledgerwatch/erigon/core/vm"
 	"github.com/ledgerwatch/erigon/eth/ethconfig"
 	"github.com/ledgerwatch/erigon/eth/stagedsync"
+	"github.com/ledgerwatch/erigon/params"
 	"github.com/ledgerwatch/erigon/turbo/shards"
 	stages2 "github.com/ledgerwatch/erigon/turbo/stages"
 	"github.com/ledgerwatch/erigon/zk/sequencer"
@@ -26,7 +35,36 @@ func newSyncZk(ctx context.Context, db kv.RwDB) (consensus.Engine, *vm.Config, *
 
 	vmConfig := &vm.Config{}
 
-	genesis := core.GenesisBlockByChainName(chain)
+	var genesis *types.Genesis
+
+	if strings.HasPrefix(chain, "dynamic") {
+		if config == "" {
+			panic("Config file is required for dynamic chain")
+		}
+
+		params.DynamicChainConfigPath = filepath.Dir(config)
+		genesis = core.GenesisBlockByChainName(chain)
+		filename := path.Join(params.DynamicChainConfigPath, chain+"-conf.json")
+
+		dConf := utils.DynamicConfig{}
+
+		if _, err := os.Stat(filename); err == nil {
+			dConfBytes, err := os.ReadFile(filename)
+			if err != nil {
+				panic(err)
+			}
+			if err := json.Unmarshal(dConfBytes, &dConf); err != nil {
+				panic(err)
+			}
+		}
+
+		genesis.Timestamp = dConf.Timestamp
+		genesis.GasLimit = dConf.GasLimit
+		genesis.Difficulty = big.NewInt(dConf.Difficulty)
+	} else {
+		genesis = core.GenesisBlockByChainName(chain)
+	}
+
 	chainConfig, genesisBlock, genesisErr := core.CommitGenesisBlock(db, genesis, "")
 	if _, ok := genesisErr.(*chain3.ConfigCompatError); genesisErr != nil && !ok {
 		panic(genesisErr)

cmd/rpcdaemon/cli/config.go

@@ -96,6 +96,9 @@ func RootCommand() (*cobra.Command, *httpcfg.HttpCfg) {
 	rootCmd.PersistentFlags().Uint64Var(&cfg.MaxTraces, "trace.maxtraces", 200, "Sets a limit on traces that can be returned in trace_filter")
 	rootCmd.PersistentFlags().BoolVar(&cfg.WebsocketEnabled, "ws", false, "Enable Websockets - Same port as HTTP")
 	rootCmd.PersistentFlags().BoolVar(&cfg.WebsocketCompression, "ws.compression", false, "Enable Websocket compression (RFC 7692)")
+	rootCmd.PersistentFlags().StringVar(&cfg.WebSocketListenAddress, "ws.addr", nodecfg.DefaultHTTPHost, "Websocket server listening interface")
+	rootCmd.PersistentFlags().IntVar(&cfg.WebSocketPort, "ws.port", nodecfg.DefaultHTTPPort, "Websocket server listening port")
+	rootCmd.PersistentFlags().StringSliceVar(&cfg.WebsocketCORSDomain, "ws.corsdomain", []string{}, "Comma separated list of domains from which to accept cross origin requests (browser enforced)")
 	rootCmd.PersistentFlags().StringVar(&cfg.RpcAllowListFilePath, utils.RpcAccessListFlag.Name, "", "Specify granular (method-by-method) API allowlist")
 	rootCmd.PersistentFlags().UintVar(&cfg.RpcBatchConcurrency, utils.RpcBatchConcurrencyFlag.Name, 2, utils.RpcBatchConcurrencyFlag.Usage)
 	rootCmd.PersistentFlags().BoolVar(&cfg.RpcStreamingDisable, utils.RpcStreamingDisableFlag.Name, false, utils.RpcStreamingDisableFlag.Usage)
@@ -533,14 +536,10 @@ func startRegularRpcServer(ctx context.Context, cfg httpcfg.HttpCfg, rpcAPI []rp
 	}
 
 	httpHandler := node.NewHTTPHandlerStack(srv, cfg.HttpCORSDomain, cfg.HttpVirtualHost, cfg.HttpCompression)
-	var wsHandler http.Handler
-	if cfg.WebsocketEnabled {
-		wsHandler = srv.WebsocketHandler([]string{"*"}, nil, cfg.WebsocketCompression)
-	}
 
 	graphQLHandler := graphql.CreateHandler(defaultAPIList)
 
-	apiHandler, err := createHandler(cfg, defaultAPIList, httpHandler, wsHandler, graphQLHandler, nil)
+	apiHandler, err := createHandler(cfg, defaultAPIList, httpHandler, nil, graphQLHandler, nil)
 	if err != nil {
 		return err
 	}
@@ -609,6 +608,69 @@ func startRegularRpcServer(ctx context.Context, cfg httpcfg.HttpCfg, rpcAPI []rp
 			log.Info("GRPC endpoint closed", "url", grpcEndpoint)
 		}
 	}()
+
+	if cfg.WebsocketEnabled {
+		wsSrv := rpc.NewServer(cfg.RpcBatchConcurrency, cfg.TraceRequests, cfg.RpcStreamingDisable)
+
+		allowListForRPC, err := parseAllowListForRPC(cfg.RpcAllowListFilePath)
+		if err != nil {
+			return err
+		}
+
+		var wsApiFlags []string
+		for _, flag := range cfg.WebSocketApi {
+			if flag != "engine" {
+				wsApiFlags = append(wsApiFlags, flag)
+			}
+		}
+
+		if err := node.RegisterApisFromWhitelist(defaultAPIList, wsApiFlags, wsSrv, false); err != nil {
+			return fmt.Errorf("could not start register WS apis: %w", err)
+		}
+		wsSrv.SetAllowList(allowListForRPC)
+
+		wsSrv.SetBatchLimit(cfg.BatchLimit)
+
+		var defaultAPIList []rpc.API
+
+		for _, api := range rpcAPI {
+			if api.Namespace != "engine" {
+				defaultAPIList = append(defaultAPIList, api)
+			}
+		}
+
+		var apiFlags []string
+		for _, flag := range cfg.API {
+			if flag != "engine" {
+				apiFlags = append(apiFlags, flag)
+			}
+		}
+
+		if err := node.RegisterApisFromWhitelist(defaultAPIList, apiFlags, wsSrv, false); err != nil {
+			return fmt.Errorf("could not start register RPC apis: %w", err)
+		}
+
+		wsEndpoint := fmt.Sprintf("%s:%d", cfg.WebSocketListenAddress, cfg.WebSocketPort)
+
+		wsHttpHandler := wsSrv.WebsocketHandler(cfg.WebsocketCORSDomain, nil, cfg.WebsocketCompression)
+		wsHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			wsHttpHandler.ServeHTTP(w, r)
+		})
+
+		wsListener, wsHttpAddr, err := node.StartHTTPEndpoint(wsEndpoint, cfg.HTTPTimeouts, wsHandler)
+		if err != nil {
+			return fmt.Errorf("could not start ws RPC api: %w", err)
+		}
+
+		defer func() {
+			wsSrv.Stop()
+			shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			_ = wsListener.Shutdown(shutdownCtx)
+			log.Info("WS endpoint closed", "url", wsHttpAddr)
+		}()
+	}
+
 	<-ctx.Done()
 	log.Info("Exiting...")
 	return nil
@@ -685,7 +747,7 @@ func obtainJWTSecret(cfg httpcfg.HttpCfg) ([]byte, error) {
 	return jwtSecret, nil
 }
 
-func createHandler(cfg httpcfg.HttpCfg, apiList []rpc.API, httpHandler http.Handler, wsHandler http.Handler, graphQLHandler http.Handler, jwtSecret []byte) (http.Handler, error) {
+func createHandler(cfg httpcfg.HttpCfg, apiList []rpc.API, httpHandler, wsHandler, graphQLHandler http.Handler, jwtSecret []byte) (http.Handler, error) {
 	var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if cfg.GraphQLEnabled && graphql.ProcessGraphQLcheckIfNeeded(graphQLHandler, w, r) {
 			return

cmd/rpcdaemon/cli/httpcfg/http_cfg.go

@@ -32,15 +32,20 @@ type HttpCfg struct {
 	MaxTraces                uint64
 	WebsocketEnabled         bool
 	WebsocketCompression     bool
-	RpcAllowListFilePath     string
-	RpcBatchConcurrency      uint
-	RpcStreamingDisable      bool
-	DBReadConcurrency        int
-	TraceCompatibility       bool // Bug for bug compatibility for trace_ routines with OpenEthereum
-	TxPoolApiAddr            string
-	StateCache               kvcache.CoherentConfig
-	Snap                     ethconfig.Snapshot
-	Sync                     ethconfig.Sync
+	WebSocketListenAddress   string
+	WebSocketPort            int
+	WebsocketCORSDomain      []string
+	WebSocketApi             []string
+
+	RpcAllowListFilePath string
+	RpcBatchConcurrency  uint
+	RpcStreamingDisable  bool
+	DBReadConcurrency    int
+	TraceCompatibility   bool // Bug for bug compatibility for trace_ routines with OpenEthereum
+	TxPoolApiAddr        string
+	StateCache           kvcache.CoherentConfig
+	Snap                 ethconfig.Snapshot
+	Sync                 ethconfig.Sync
 
 	// GRPC server
 	GRPCServerEnabled      bool