Skip to content

Commit

Permalink
cmd/evm: benchmarking via statetest command + filter by name, index…
Browse files Browse the repository at this point in the history 
… and fork (ethereum#30442)

When `evm statetest --bench` is specified, benchmark the execution
similarly to `evm run`.

Also adds the ability to filter tests by name, index and fork. 

---------

Co-authored-by: Martin Holst Swende <[email protected]>
  • Loading branch information
@jwasinger @holiman
jwasinger and holiman authored Nov 8, 2024
1 parent 5b78aef commit d42d450
Show file tree
Hide file tree
Showing 3 changed files with 144 additions and 61 deletions.
64 changes: 41 additions & 23 deletions cmd/evm/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,36 +76,53 @@ func readGenesis(genesisPath string) *core.Genesis {
}

type execStats struct {
time time.Duration // The execution time.
allocs int64 // The number of heap allocations during execution.
bytesAllocated int64 // The cumulative number of bytes allocated during execution.
Time time.Duration `json:"time"` // The execution Time.
Allocs int64 `json:"allocs"` // The number of heap allocations during execution.
BytesAllocated int64 `json:"bytesAllocated"` // The cumulative number of bytes allocated during execution.
GasUsed uint64 `json:"gasUsed"` // the amount of gas used during execution
}

func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []byte, gasLeft uint64, stats execStats, err error) {
func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) ([]byte, execStats, error) {
if bench {
// Do one warm-up run
output, gasUsed, err := execFunc()
result := testing.Benchmark(func(b *testing.B) {
for i := 0; i < b.N; i++ {
output, gasLeft, err = execFunc()
haveOutput, haveGasUsed, haveErr := execFunc()
if !bytes.Equal(haveOutput, output) {
b.Fatalf("output differs, have\n%x\nwant%x\n", haveOutput, output)
}
if haveGasUsed != gasUsed {
b.Fatalf("gas differs, have %v want%v", haveGasUsed, gasUsed)
}
if haveErr != err {
b.Fatalf("err differs, have %v want%v", haveErr, err)
}
}
})

// Get the average execution time from the benchmarking result.
// There are other useful stats here that could be reported.
stats.time = time.Duration(result.NsPerOp())
stats.allocs = result.AllocsPerOp()
stats.bytesAllocated = result.AllocedBytesPerOp()
} else {
var memStatsBefore, memStatsAfter goruntime.MemStats
goruntime.ReadMemStats(&memStatsBefore)
startTime := time.Now()
output, gasLeft, err = execFunc()
stats.time = time.Since(startTime)
goruntime.ReadMemStats(&memStatsAfter)
stats.allocs = int64(memStatsAfter.Mallocs - memStatsBefore.Mallocs)
stats.bytesAllocated = int64(memStatsAfter.TotalAlloc - memStatsBefore.TotalAlloc)
stats := execStats{
Time: time.Duration(result.NsPerOp()),
Allocs: result.AllocsPerOp(),
BytesAllocated: result.AllocedBytesPerOp(),
GasUsed: gasUsed,
}
return output, stats, err
}

return output, gasLeft, stats, err
var memStatsBefore, memStatsAfter goruntime.MemStats
goruntime.ReadMemStats(&memStatsBefore)
t0 := time.Now()
output, gasUsed, err := execFunc()
duration := time.Since(t0)
goruntime.ReadMemStats(&memStatsAfter)
stats := execStats{
Time: duration,
Allocs: int64(memStatsAfter.Mallocs - memStatsBefore.Mallocs),
BytesAllocated: int64(memStatsAfter.TotalAlloc - memStatsBefore.TotalAlloc),
GasUsed: gasUsed,
}
return output, stats, err
}

func runCmd(ctx *cli.Context) error {
Expand Down Expand Up @@ -265,12 +282,13 @@ func runCmd(ctx *cli.Context) error {
statedb.SetCode(receiver, code)
}
execFunc = func() ([]byte, uint64, error) {
return runtime.Call(receiver, input, &runtimeConfig)
output, gasLeft, err := runtime.Call(receiver, input, &runtimeConfig)
return output, initialGas - gasLeft, err
}
}

bench := ctx.Bool(BenchFlag.Name)
output, leftOverGas, stats, err := timedExec(bench, execFunc)
output, stats, err := timedExec(bench, execFunc)

if ctx.Bool(DumpFlag.Name) {
root, err := statedb.Commit(genesisConfig.Number, true)
Expand Down Expand Up @@ -300,7 +318,7 @@ func runCmd(ctx *cli.Context) error {
execution time: %v
allocations: %d
allocated bytes: %d
`, initialGas-leftOverGas, stats.time, stats.allocs, stats.bytesAllocated)
`, stats.GasUsed, stats.Time, stats.Allocs, stats.BytesAllocated)
}
if tracer == nil {
fmt.Printf("%#x\n", output)
Expand Down
125 changes: 95 additions & 30 deletions cmd/evm/staterunner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,26 +27,51 @@ import (
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/tracers/logger"
"github.com/ethereum/go-ethereum/internal/flags"
"github.com/ethereum/go-ethereum/tests"
"github.com/urfave/cli/v2"
)

var (
forkFlag = &cli.StringFlag{
Name: "statetest.fork",
Usage: "The hard-fork to run the test against",
Category: flags.VMCategory,
}
idxFlag = &cli.IntFlag{
Name: "statetest.index",
Usage: "The index of the subtest to run",
Category: flags.VMCategory,
Value: -1, // default to select all subtest indices
}
testNameFlag = &cli.StringFlag{
Name: "statetest.name",
Usage: "The name of the state test to run",
Category: flags.VMCategory,
}
)
var stateTestCommand = &cli.Command{
Action: stateTestCmd,
Name: "statetest",
Usage: "Executes the given state tests. Filenames can be fed via standard input (batch mode) or as an argument (one-off execution).",
ArgsUsage: "<file>",
Flags: []cli.Flag{
forkFlag,
idxFlag,
testNameFlag,
},
}

// StatetestResult contains the execution status after running a state test, any
// error that might have occurred and a dump of the final state if requested.
type StatetestResult struct {
Name string `json:"name"`
Pass bool `json:"pass"`
Root *common.Hash `json:"stateRoot,omitempty"`
Fork string `json:"fork"`
Error string `json:"error,omitempty"`
State *state.Dump `json:"state,omitempty"`
Name string `json:"name"`
Pass bool `json:"pass"`
Root *common.Hash `json:"stateRoot,omitempty"`
Fork string `json:"fork"`
Error string `json:"error,omitempty"`
State *state.Dump `json:"state,omitempty"`
BenchStats *execStats `json:"benchStats,omitempty"`
}

func stateTestCmd(ctx *cli.Context) error {
Expand All @@ -67,7 +92,7 @@ func stateTestCmd(ctx *cli.Context) error {
}
// Load the test content from the input file
if len(ctx.Args().First()) != 0 {
return runStateTest(ctx.Args().First(), cfg, ctx.Bool(DumpFlag.Name))
return runStateTest(ctx, ctx.Args().First(), cfg, ctx.Bool(DumpFlag.Name), ctx.Bool(BenchFlag.Name))
}
// Read filenames from stdin and execute back-to-back
scanner := bufio.NewScanner(os.Stdin)
Expand All @@ -76,15 +101,48 @@ func stateTestCmd(ctx *cli.Context) error {
if len(fname) == 0 {
return nil
}
if err := runStateTest(fname, cfg, ctx.Bool(DumpFlag.Name)); err != nil {
if err := runStateTest(ctx, fname, cfg, ctx.Bool(DumpFlag.Name), ctx.Bool(BenchFlag.Name)); err != nil {
return err
}
}
return nil
}

type stateTestCase struct {
name string
test tests.StateTest
st tests.StateSubtest
}

// collectMatchedSubtests returns test cases which match against provided filtering CLI parameters
func collectMatchedSubtests(ctx *cli.Context, testsByName map[string]tests.StateTest) []stateTestCase {
var res []stateTestCase
subtestName := ctx.String(testNameFlag.Name)
if subtestName != "" {
if subtest, ok := testsByName[subtestName]; ok {
testsByName := make(map[string]tests.StateTest)
testsByName[subtestName] = subtest
}
}
idx := ctx.Int(idxFlag.Name)
fork := ctx.String(forkFlag.Name)

for key, test := range testsByName {
for _, st := range test.Subtests() {
if idx != -1 && st.Index != idx {
continue
}
if fork != "" && st.Fork != fork {
continue
}
res = append(res, stateTestCase{name: key, st: st, test: test})
}
}
return res
}

// runStateTest loads the state-test given by fname, and executes the test.
func runStateTest(fname string, cfg vm.Config, dump bool) error {
func runStateTest(ctx *cli.Context, fname string, cfg vm.Config, dump bool, bench bool) error {
src, err := os.ReadFile(fname)
if err != nil {
return err
Expand All @@ -94,31 +152,38 @@ func runStateTest(fname string, cfg vm.Config, dump bool) error {
return err
}

matchingTests := collectMatchedSubtests(ctx, testsByName)

// Iterate over all the tests, run them and aggregate the results
results := make([]StatetestResult, 0, len(testsByName))
for key, test := range testsByName {
for _, st := range test.Subtests() {
// Run the test and aggregate the result
result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
test.Run(st, cfg, false, rawdb.HashScheme, func(err error, tstate *tests.StateTestState) {
var root common.Hash
if tstate.StateDB != nil {
root = tstate.StateDB.IntermediateRoot(false)
result.Root = &root
fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root)
if dump { // Dump any state to aid debugging
cpy, _ := state.New(root, tstate.StateDB.Database())
dump := cpy.RawDump(nil)
result.State = &dump
}
}
if err != nil {
// Test failed, mark as so
result.Pass, result.Error = false, err.Error()
var results []StatetestResult
for _, test := range matchingTests {
// Run the test and aggregate the result
result := &StatetestResult{Name: test.name, Fork: test.st.Fork, Pass: true}
test.test.Run(test.st, cfg, false, rawdb.HashScheme, func(err error, tstate *tests.StateTestState) {
var root common.Hash
if tstate.StateDB != nil {
root = tstate.StateDB.IntermediateRoot(false)
result.Root = &root
fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root)
if dump { // Dump any state to aid debugging
cpy, _ := state.New(root, tstate.StateDB.Database())
dump := cpy.RawDump(nil)
result.State = &dump
}
}
if err != nil {
// Test failed, mark as so
result.Pass, result.Error = false, err.Error()
}
})
if bench {
_, stats, _ := timedExec(true, func() ([]byte, uint64, error) {
_, _, gasUsed, _ := test.test.RunNoVerify(test.st, cfg, false, rawdb.HashScheme)
return nil, gasUsed, nil
})
results = append(results, *result)
result.BenchStats = &stats
}
results = append(results, *result)
}
out, _ := json.MarshalIndent(results, "", " ")
fmt.Println(string(out))
Expand Down
16 changes: 8 additions & 8 deletions tests/state_test_util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ func (t *StateTest) checkError(subtest StateSubtest, err error) error {

// Run executes a specific subtest and verifies the post-state and logs
func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string, postCheck func(err error, st *StateTestState)) (result error) {
st, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter, scheme)
st, root, _, err := t.RunNoVerify(subtest, vmconfig, snapshotter, scheme)
// Invoke the callback at the end of function for further analysis.
defer func() {
postCheck(result, &st)
Expand Down Expand Up @@ -228,10 +228,10 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bo

// RunNoVerify runs a specific subtest and returns the statedb and post-state root.
// Remember to call state.Close after verifying the test result!
func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string) (st StateTestState, root common.Hash, err error) {
func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string) (st StateTestState, root common.Hash, gasUsed uint64, err error) {
config, eips, err := GetChainConfig(subtest.Fork)
if err != nil {
return st, common.Hash{}, UnsupportedForkError{subtest.Fork}
return st, common.Hash{}, 0, UnsupportedForkError{subtest.Fork}
}
vmconfig.ExtraEips = eips

Expand All @@ -250,7 +250,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
post := t.json.Post[subtest.Fork][subtest.Index]
msg, err := t.json.Tx.toMessage(post, baseFee)
if err != nil {
return st, common.Hash{}, err
return st, common.Hash{}, 0, err
}

{ // Blob transactions may be present after the Cancun fork.
Expand All @@ -260,7 +260,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
// Here, we just do this shortcut smaller fix, since state tests do not
// utilize those codepaths
if len(msg.BlobHashes)*params.BlobTxBlobGasPerBlob > params.MaxBlobGasPerBlock {
return st, common.Hash{}, errors.New("blob gas exceeds maximum")
return st, common.Hash{}, 0, errors.New("blob gas exceeds maximum")
}
}

Expand All @@ -269,10 +269,10 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
var ttx types.Transaction
err := ttx.UnmarshalBinary(post.TxBytes)
if err != nil {
return st, common.Hash{}, err
return st, common.Hash{}, 0, err
}
if _, err := types.Sender(types.LatestSigner(config), &ttx); err != nil {
return st, common.Hash{}, err
return st, common.Hash{}, 0, err
}
}

Expand Down Expand Up @@ -322,7 +322,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
receipt := &types.Receipt{GasUsed: vmRet.UsedGas}
tracer.OnTxEnd(receipt, nil)
}
return st, root, err
return st, root, vmRet.UsedGas, err
}

func (t *StateTest) gasLimit(subtest StateSubtest) uint64 {
Expand Down

0 comments on commit d42d450

Please sign in to comment.