Skip to content

Commit

Permalink
Merge branch 'master' into jord/dkg-engine-docs
Browse files Browse the repository at this point in the history
  • Loading branch information
jordanschalm authored Dec 3, 2024
2 parents 3f1de8f + 533123a commit 858da9b
Show file tree
Hide file tree
Showing 63 changed files with 3,002 additions and 470 deletions.
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,8 @@ generate-mocks: install-mock-generators
mockery --name 'API' --dir="./engine/protocol" --case=underscore --output="./engine/protocol/mock" --outpkg="mock"
mockery --name '.*' --dir="./engine/access/state_stream" --case=underscore --output="./engine/access/state_stream/mock" --outpkg="mock"
mockery --name 'BlockTracker' --dir="./engine/access/subscription" --case=underscore --output="./engine/access/subscription/mock" --outpkg="mock"
mockery --name 'DataProvider' --dir="./engine/access/rest/websockets/data_provider" --case=underscore --output="./engine/access/rest/websockets/data_provider/mock" --outpkg="mock"
mockery --name 'DataProvider' --dir="./engine/access/rest/websockets/data_providers" --case=underscore --output="./engine/access/rest/websockets/data_providers/mock" --outpkg="mock"
mockery --name 'DataProviderFactory' --dir="./engine/access/rest/websockets/data_providers" --case=underscore --output="./engine/access/rest/websockets/data_providers/mock" --outpkg="mock"
mockery --name 'ExecutionDataTracker' --dir="./engine/access/subscription" --case=underscore --output="./engine/access/subscription/mock" --outpkg="mock"
mockery --name 'ConnectionFactory' --dir="./engine/access/rpc/connection" --case=underscore --output="./engine/access/rpc/connection/mock" --outpkg="mock"
mockery --name 'Communicator' --dir="./engine/access/rpc/backend" --case=underscore --output="./engine/access/rpc/backend/mock" --outpkg="mock"
Expand Down
20 changes: 10 additions & 10 deletions access/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -1066,7 +1066,7 @@ func (h *Handler) SubscribeBlocksFromStartBlockID(request *access.SubscribeBlock
}

sub := h.api.SubscribeBlocksFromStartBlockID(stream.Context(), startBlockID, blockStatus)
return subscription.HandleSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus))
return subscription.HandleRPCSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus))
}

// SubscribeBlocksFromStartHeight handles subscription requests for blocks started from block height.
Expand All @@ -1093,7 +1093,7 @@ func (h *Handler) SubscribeBlocksFromStartHeight(request *access.SubscribeBlocks
}

sub := h.api.SubscribeBlocksFromStartHeight(stream.Context(), request.GetStartBlockHeight(), blockStatus)
return subscription.HandleSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus))
return subscription.HandleRPCSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus))
}

// SubscribeBlocksFromLatest handles subscription requests for blocks started from latest sealed block.
Expand All @@ -1120,7 +1120,7 @@ func (h *Handler) SubscribeBlocksFromLatest(request *access.SubscribeBlocksFromL
}

sub := h.api.SubscribeBlocksFromLatest(stream.Context(), blockStatus)
return subscription.HandleSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus))
return subscription.HandleRPCSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus))
}

// handleBlocksResponse handles the subscription to block updates and sends
Expand Down Expand Up @@ -1179,7 +1179,7 @@ func (h *Handler) SubscribeBlockHeadersFromStartBlockID(request *access.Subscrib
}

sub := h.api.SubscribeBlockHeadersFromStartBlockID(stream.Context(), startBlockID, blockStatus)
return subscription.HandleSubscription(sub, h.handleBlockHeadersResponse(stream.Send))
return subscription.HandleRPCSubscription(sub, h.handleBlockHeadersResponse(stream.Send))
}

// SubscribeBlockHeadersFromStartHeight handles subscription requests for block headers started from block height.
Expand All @@ -1206,7 +1206,7 @@ func (h *Handler) SubscribeBlockHeadersFromStartHeight(request *access.Subscribe
}

sub := h.api.SubscribeBlockHeadersFromStartHeight(stream.Context(), request.GetStartBlockHeight(), blockStatus)
return subscription.HandleSubscription(sub, h.handleBlockHeadersResponse(stream.Send))
return subscription.HandleRPCSubscription(sub, h.handleBlockHeadersResponse(stream.Send))
}

// SubscribeBlockHeadersFromLatest handles subscription requests for block headers started from latest sealed block.
Expand All @@ -1233,7 +1233,7 @@ func (h *Handler) SubscribeBlockHeadersFromLatest(request *access.SubscribeBlock
}

sub := h.api.SubscribeBlockHeadersFromLatest(stream.Context(), blockStatus)
return subscription.HandleSubscription(sub, h.handleBlockHeadersResponse(stream.Send))
return subscription.HandleRPCSubscription(sub, h.handleBlockHeadersResponse(stream.Send))
}

// handleBlockHeadersResponse handles the subscription to block updates and sends
Expand Down Expand Up @@ -1293,7 +1293,7 @@ func (h *Handler) SubscribeBlockDigestsFromStartBlockID(request *access.Subscrib
}

sub := h.api.SubscribeBlockDigestsFromStartBlockID(stream.Context(), startBlockID, blockStatus)
return subscription.HandleSubscription(sub, h.handleBlockDigestsResponse(stream.Send))
return subscription.HandleRPCSubscription(sub, h.handleBlockDigestsResponse(stream.Send))
}

// SubscribeBlockDigestsFromStartHeight handles subscription requests for lightweight blocks started from block height.
Expand All @@ -1320,7 +1320,7 @@ func (h *Handler) SubscribeBlockDigestsFromStartHeight(request *access.Subscribe
}

sub := h.api.SubscribeBlockDigestsFromStartHeight(stream.Context(), request.GetStartBlockHeight(), blockStatus)
return subscription.HandleSubscription(sub, h.handleBlockDigestsResponse(stream.Send))
return subscription.HandleRPCSubscription(sub, h.handleBlockDigestsResponse(stream.Send))
}

// SubscribeBlockDigestsFromLatest handles subscription requests for lightweight block started from latest sealed block.
Expand All @@ -1347,7 +1347,7 @@ func (h *Handler) SubscribeBlockDigestsFromLatest(request *access.SubscribeBlock
}

sub := h.api.SubscribeBlockDigestsFromLatest(stream.Context(), blockStatus)
return subscription.HandleSubscription(sub, h.handleBlockDigestsResponse(stream.Send))
return subscription.HandleRPCSubscription(sub, h.handleBlockDigestsResponse(stream.Send))
}

// handleBlockDigestsResponse handles the subscription to block updates and sends
Expand Down Expand Up @@ -1433,7 +1433,7 @@ func (h *Handler) SendAndSubscribeTransactionStatuses(
sub := h.api.SubscribeTransactionStatuses(ctx, &tx, request.GetEventEncodingVersion())

messageIndex := counters.NewMonotonousCounter(0)
return subscription.HandleSubscription(sub, func(txResults []*TransactionResult) error {
return subscription.HandleRPCSubscription(sub, func(txResults []*TransactionResult) error {
for i := range txResults {
index := messageIndex.Value()
if ok := messageIndex.Set(index + 1); !ok {
Expand Down
32 changes: 4 additions & 28 deletions cmd/scaffold.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@ import (
"github.com/onflow/flow-go/cmd/build"
"github.com/onflow/flow-go/config"
"github.com/onflow/flow-go/consensus/hotstuff/persister"
"github.com/onflow/flow-go/fvm"
"github.com/onflow/flow-go/fvm/environment"
"github.com/onflow/flow-go/fvm/initialize"
"github.com/onflow/flow-go/model/flow"
"github.com/onflow/flow-go/model/flow/filter"
"github.com/onflow/flow-go/module"
Expand Down Expand Up @@ -1522,32 +1521,9 @@ func (fnb *FlowNodeBuilder) initLocal() error {
}

func (fnb *FlowNodeBuilder) initFvmOptions() {
blockFinder := environment.NewBlockFinder(fnb.Storage.Headers)
vmOpts := []fvm.Option{
fvm.WithChain(fnb.RootChainID.Chain()),
fvm.WithBlocks(blockFinder),
fvm.WithAccountStorageLimit(true),
}
switch fnb.RootChainID {
case flow.Testnet,
flow.Sandboxnet,
flow.Previewnet,
flow.Mainnet:
vmOpts = append(vmOpts,
fvm.WithTransactionFeesEnabled(true),
)
}
switch fnb.RootChainID {
case flow.Testnet,
flow.Sandboxnet,
flow.Previewnet,
flow.Localnet,
flow.Benchnet:
vmOpts = append(vmOpts,
fvm.WithContractDeploymentRestricted(false),
)
}
fnb.FvmOptions = vmOpts
fnb.FvmOptions = initialize.InitFvmOptions(
fnb.RootChainID, fnb.Storage.Headers,
)
}

// handleModules initializes the given module.
Expand Down
4 changes: 4 additions & 0 deletions cmd/util/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ import (
"github.com/onflow/flow-go/cmd/util/cmd/snapshot"
system_addresses "github.com/onflow/flow-go/cmd/util/cmd/system-addresses"
truncate_database "github.com/onflow/flow-go/cmd/util/cmd/truncate-database"
verify_evm_offchain_replay "github.com/onflow/flow-go/cmd/util/cmd/verify-evm-offchain-replay"
verify_execution_result "github.com/onflow/flow-go/cmd/util/cmd/verify_execution_result"
"github.com/onflow/flow-go/cmd/util/cmd/version"
"github.com/onflow/flow-go/module/profiler"
)
Expand Down Expand Up @@ -126,6 +128,8 @@ func addCommands() {
rootCmd.AddCommand(debug_script.Cmd)
rootCmd.AddCommand(generate_authorization_fixes.Cmd)
rootCmd.AddCommand(evm_state_exporter.Cmd)
rootCmd.AddCommand(verify_execution_result.Cmd)
rootCmd.AddCommand(verify_evm_offchain_replay.Cmd)
}

func initConfig() {
Expand Down
88 changes: 88 additions & 0 deletions cmd/util/cmd/verify-evm-offchain-replay/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package verify

import (
"fmt"
"strconv"
"strings"

"github.com/rs/zerolog/log"
"github.com/spf13/cobra"

"github.com/onflow/flow-go/model/flow"
)

var (
flagDatadir string
flagExecutionDataDir string
flagEVMStateGobDir string
flagChain string
flagFromTo string
flagSaveEveryNBlocks uint64
)

// usage example
//
// ./util verify-evm-offchain-replay --chain flow-testnet --from_to 211176670-211177000
// --datadir /var/flow/data/protocol --execution_data_dir /var/flow/data/execution_data
var Cmd = &cobra.Command{
Use: "verify-evm-offchain-replay",
Short: "verify evm offchain replay with execution data",
Run: run,
}

func init() {
Cmd.Flags().StringVar(&flagChain, "chain", "", "Chain name")
_ = Cmd.MarkFlagRequired("chain")

Cmd.Flags().StringVar(&flagDatadir, "datadir", "/var/flow/data/protocol",
"directory that stores the protocol state")

Cmd.Flags().StringVar(&flagExecutionDataDir, "execution_data_dir", "/var/flow/data/execution_data",
"directory that stores the execution state")

Cmd.Flags().StringVar(&flagFromTo, "from_to", "",
"the flow height range to verify blocks, i.e, 1-1000, 1000-2000, 2000-3000, etc.")

Cmd.Flags().StringVar(&flagEVMStateGobDir, "evm_state_gob_dir", "/var/flow/data/evm_state_gob",
"directory that stores the evm state gob files as checkpoint")

Cmd.Flags().Uint64Var(&flagSaveEveryNBlocks, "save_every", uint64(1_000_000),
"save the evm state gob files every N blocks")
}

func run(*cobra.Command, []string) {
chainID := flow.ChainID(flagChain)

from, to, err := parseFromTo(flagFromTo)
if err != nil {
log.Fatal().Err(err).Msg("could not parse from_to")
}

err = Verify(log.Logger, from, to, chainID, flagDatadir, flagExecutionDataDir, flagEVMStateGobDir, flagSaveEveryNBlocks)
if err != nil {
log.Fatal().Err(err).Msg("could not verify height")
}
}

func parseFromTo(fromTo string) (from, to uint64, err error) {
parts := strings.Split(fromTo, "-")
if len(parts) != 2 {
return 0, 0, fmt.Errorf("invalid format: expected 'from-to', got '%s'", fromTo)
}

from, err = strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 64)
if err != nil {
return 0, 0, fmt.Errorf("invalid 'from' value: %w", err)
}

to, err = strconv.ParseUint(strings.TrimSpace(parts[1]), 10, 64)
if err != nil {
return 0, 0, fmt.Errorf("invalid 'to' value: %w", err)
}

if from > to {
return 0, 0, fmt.Errorf("'from' value (%d) must be less than or equal to 'to' value (%d)", from, to)
}

return from, to, nil
}
Loading

0 comments on commit 858da9b

Please sign in to comment.