Merge branch 'master' into tarak/5077-validate-bootstarpping-keys

.github/workflows/cd.yml

@@ -0,0 +1,31 @@
+name: CD
+
+on:
+  push:
+    tags: 
+      - '*'
+
+jobs:
+  docker-push:
+    name: Push to container registry
+    runs-on: ubuntu-latest
+    steps:
+    - name: Granting private modules access
+      run: |
+        git config --global url."https://${{ secrets.GO_MODULES_TOKEN }}:[email protected]/onflow".insteadOf "https://github.com/onflow"
+    - name: Setup Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: '1.13'
+    - name: Checkout repo
+      uses: actions/checkout@v2
+    - name: Install SSH agent
+      uses: webfactory/[email protected]
+      with:
+        ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - name: Build relic
+      run: make crypto/relic/build
+    - name: Docker build
+      run: make docker-build-flow
+    - name: Docker push
+      run: make docker-push-flow

.github/workflows/ci.yml

@@ -36,7 +36,7 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v2
       with:
-        go-version: '1.14'
+        go-version: '1.13'
     - name: Checkout repo
       uses: actions/checkout@v2
     - name: Run tests
@@ -51,7 +51,7 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v2
       with:
-        go-version: '1.14'
+        go-version: '1.13'
     - name: Checkout repo
       uses: actions/checkout@v2
     - name: Install SSH agent

Makefile

@@ -24,8 +24,7 @@ UNAME := $(shell uname)
 K8S_YAMLS_LOCATION_STAGING=./k8s/staging
 
 # docker container registry
-CONTAINER_REGISTRY=gcr.io/dl-flow
-
+export CONTAINER_REGISTRY := gcr.io/flow-container-registry
 export DOCKER_BUILDKIT := 1
 
 .PHONY: crypto/relic
@@ -115,6 +114,7 @@ generate-mocks:
 	GO111MODULE=on mockery -name '.*' -dir=fvm -case=underscore -output="./fvm/mock" -outpkg="mock"
 	GO111MODULE=on mockery -name '.*' -dir=network/gossip/libp2p/middleware -case=underscore -output="./network/gossip/libp2p/mock" -outpkg="mock"
 	GO111MODULE=on mockery -name 'Connector' -dir=network/gossip/libp2p -case=underscore -output="./network/gossip/libp2p/mock" -outpkg="mock"
+	GO111MODULE=on mockery -name 'SubscriptionManager' -dir=network/gossip/libp2p/channel -case=underscore -output="./network/gossip/libp2p/mock" -outpkg="mock"
 	GO111MODULE=on mockery -name 'Vertex' -dir="./consensus/hotstuff/forks/finalizer/forest" -case=underscore -output="./consensus/hotstuff/forks/finalizer/forest/mock" -outpkg="mock"
 	GO111MODULE=on mockery -name '.*' -dir="./consensus/hotstuff" -case=underscore -output="./consensus/hotstuff/mocks" -outpkg="mocks"
 	GO111MODULE=on mockery -name '.*' -dir="./engine/access/wrapper" -case=underscore -output="./engine/access/mock" -outpkg="mock"

access/legacy/convert/convert.go

@@ -77,7 +77,7 @@ func MessageToTransaction(m *entitiesproto.Transaction, chain flow.Chain) (flow.
 func TransactionToMessage(tb flow.TransactionBody) *entitiesproto.Transaction {
 	proposalKeyMessage := &entitiesproto.Transaction_ProposalKey{
 		Address:        tb.ProposalKey.Address.Bytes(),
-		KeyId:          uint32(tb.ProposalKey.KeyID),
+		KeyId:          uint32(tb.ProposalKey.KeyIndex),
 		SequenceNumber: tb.ProposalKey.SequenceNumber,
 	}
 
@@ -91,7 +91,7 @@ func TransactionToMessage(tb flow.TransactionBody) *entitiesproto.Transaction {
 	for i, sig := range tb.PayloadSignatures {
 		payloadSigMessages[i] = &entitiesproto.Transaction_Signature{
 			Address:   sig.Address.Bytes(),
-			KeyId:     uint32(sig.KeyID),
+			KeyId:     uint32(sig.KeyIndex),
 			Signature: sig.Signature,
 		}
 	}
@@ -101,7 +101,7 @@ func TransactionToMessage(tb flow.TransactionBody) *entitiesproto.Transaction {
 	for i, sig := range tb.EnvelopeSignatures {
 		envelopeSigMessages[i] = &entitiesproto.Transaction_Signature{
 			Address:   sig.Address.Bytes(),
-			KeyId:     uint32(sig.KeyID),
+			KeyId:     uint32(sig.KeyIndex),
 			Signature: sig.Signature,
 		}
 	}
@@ -231,7 +231,7 @@ func AccountToMessage(a *flow.Account) (*entitiesproto.Account, error) {
 	return &entitiesproto.Account{
 		Address: a.Address.Bytes(),
 		Balance: a.Balance,
-		Code:    a.Code,
+		Code:    nil,
 		Keys:    keys,
 	}, nil
 }

cmd/execution/main.go

@@ -98,7 +98,7 @@ func main() {
 			rt := runtime.NewInterpreterRuntime()
 
 			vm := fvm.New(rt)
-			vmCtx := fvm.NewContext(node.FvmOptions...)
+			vmCtx := fvm.NewContext(node.Logger, node.FvmOptions...)
 
 			manager, err := computation.New(
 				node.Logger,
@@ -163,8 +163,8 @@ func main() {
 				// if execution database has been bootstrapped, then the root statecommit must equal to the one
 				// in the bootstrap folder
 				if !bytes.Equal(commit, node.RootSeal.FinalState) {
-					return nil, fmt.Errorf("mismatching root statecommitment. database has state commitment: %v, "+
-						"bootstap has statecommitment: %v",
+					return nil, fmt.Errorf("mismatching root statecommitment. database has state commitment: %x, "+
+						"bootstap has statecommitment: %x",
 						commit, node.RootSeal.FinalState)
 				}
 			}

cmd/scaffold.go

@@ -176,28 +176,31 @@ func (fnb *FlowNodeBuilder) enqueueNetworkInit() {
 		}
 		fnb.Middleware = mw
 
-		nodeID, err := fnb.State.Final().Identity(fnb.Me.NodeID())
-		if err != nil {
-			return nil, fmt.Errorf("could not get node id: %w", err)
-		}
-		nodeRole := nodeID.Role
-
 		participants, err := fnb.State.Final().Identities(libp2p.NetworkingSetFilter)
 		if err != nil {
 			return nil, fmt.Errorf("could not get network identities: %w", err)
 		}
 
-		var nodeTopology topology.Topology
-		if nodeRole == flow.RoleCollection {
-			nodeTopology, err = topology.NewCollectionTopology(nodeID.NodeID, fnb.State)
-		} else {
-			nodeTopology, err = topology.NewRandPermTopology(nodeRole, nodeID.NodeID)
-		}
+		// creates topology, topology manager, and subscription managers
+		//
+		// topology
+		// subscription manager
+		subscriptionManager := libp2p.NewChannelSubscriptionManager(fnb.Middleware)
+		top, err := topology.NewTopicBasedTopology(fnb.NodeID, fnb.Logger, fnb.State, subscriptionManager)
 		if err != nil {
 			return nil, fmt.Errorf("could not create topology: %w", err)
 		}
 
-		net, err := libp2p.NewNetwork(fnb.Logger, codec, participants, fnb.Me, fnb.Middleware, 10e6, nodeTopology, fnb.Metrics.Network)
+		// creates network instance
+		net, err := libp2p.NewNetwork(fnb.Logger,
+			codec,
+			participants,
+			fnb.Me,
+			fnb.Middleware,
+			10e6,
+			top,
+			subscriptionManager,
+			fnb.Metrics.Network)
 		if err != nil {
 			return nil, fmt.Errorf("could not initialize network: %w", err)
 		}

cmd/util/cmd/exec-data-json-export/transaction_exporter.go

@@ -32,7 +32,7 @@ type transactionInContext struct {
 	ProposalSequenceNumber uint64   `json:"proposal_sequence_number"`
 	AuthorizersAddressHex  []string `json:"authorizers_address_hex"`
 	EnvelopeSize           int      `json:"envelope_size"`
-	// Address(hex) + ~ + SignerIndex + ~ + KeyID + ~ + Signature(hex)
+	// Address(hex) + ~ + SignerIndex + ~ + KeyIndex + ~ + Signature(hex)
 	CompactPayloadSignatures  []string `json:"compact_payload_signatures"`
 	CompactEnvelopeSignatures []string `json:"compact_envelope_signatures"`
 	// ErrorMessage              string   `json:"error_message"`
@@ -102,13 +102,13 @@ func ExportExecutedTransactions(blockID flow.Identifier, dbPath string, outputPa
 
 				psig := make([]string, 0)
 				for _, s := range tx.PayloadSignatures {
-					cs := hex.EncodeToString(s.Address[:]) + "~" + fmt.Sprint(s.SignerIndex) + "~" + fmt.Sprint(s.KeyID) + "~" + hex.EncodeToString(s.Signature)
+					cs := hex.EncodeToString(s.Address[:]) + "~" + fmt.Sprint(s.SignerIndex) + "~" + fmt.Sprint(s.KeyIndex) + "~" + hex.EncodeToString(s.Signature)
 					psig = append(psig, cs)
 				}
 
 				esig := make([]string, 0)
 				for _, s := range tx.EnvelopeSignatures {
-					cs := hex.EncodeToString(s.Address[:]) + "~" + fmt.Sprint(s.SignerIndex) + "~" + fmt.Sprint(s.KeyID) + "~" + hex.EncodeToString(s.Signature)
+					cs := hex.EncodeToString(s.Address[:]) + "~" + fmt.Sprint(s.SignerIndex) + "~" + fmt.Sprint(s.KeyIndex) + "~" + hex.EncodeToString(s.Signature)
 					esig = append(esig, cs)
 				}
 
@@ -125,7 +125,7 @@ func ExportExecutedTransactions(blockID flow.Identifier, dbPath string, outputPa
 					GasLimit:                  tx.GasLimit,
 					PayerAddressHex:           hex.EncodeToString(tx.Payer[:]),
 					ProposalKeyAddressHex:     hex.EncodeToString(tx.ProposalKey.Address[:]),
-					ProposalKeyID:             tx.ProposalKey.KeyID,
+					ProposalKeyID:             tx.ProposalKey.KeyIndex,
 					ProposalSequenceNumber:    tx.ProposalKey.SequenceNumber,
 					AuthorizersAddressHex:     auths,
 					EnvelopeSize:              computeEnvelopeSize(tx),

cmd/util/cmd/execution-state-extract/cmd.go

@@ -16,6 +16,7 @@ var (
 	flagExecutionStateDir string
 	flagOutputDir         string
 	flagBlockHash         string
+	flagStateCommitment   string
 	flagDatadir           string
 )
 
@@ -34,37 +35,54 @@ func init() {
 		"Directory to write new Execution State to")
 	_ = Cmd.MarkFlagRequired("output-dir")
 
+	Cmd.Flags().StringVar(&flagStateCommitment, "state-commitment", "",
+		"state commitment (hex-encoded, 64 characters)")
+
 	Cmd.Flags().StringVar(&flagBlockHash, "block-hash", "",
 		"Block hash (hex-encoded, 64 characters)")
-	_ = Cmd.MarkFlagRequired("block-hash")
 
 	Cmd.Flags().StringVar(&flagDatadir, "datadir", "",
 		"directory that stores the protocol state")
-	_ = Cmd.MarkFlagRequired("datadir")
 }
 
 func run(*cobra.Command, []string) {
+	var stateCommitment []byte
 
-	blockID, err := flow.HexStringToIdentifier(flagBlockHash)
-	if err != nil {
-		log.Fatal().Err(err).Msg("malformed block hash")
+	if len(flagBlockHash) > 0 && len(flagStateCommitment) > 0 {
+		log.Fatal().Msg("cannot run the command with both block hash and state commitment as inputs, only one of them should be provided")
+		return
 	}
 
-	db := common.InitStorage(flagDatadir)
-	defer db.Close()
+	if len(flagBlockHash) > 0 {
+		blockID, err := flow.HexStringToIdentifier(flagBlockHash)
+		if err != nil {
+			log.Fatal().Err(err).Msg("malformed block hash")
+		}
 
-	cache := &metrics.NoopCollector{}
-	commits := badger.NewCommits(cache, db)
+		db := common.InitStorage(flagDatadir)
+		defer db.Close()
 
-	stateCommitment, err := getStateCommitment(commits, blockID)
-	if err != nil {
-		log.Fatal().Err(err).Msg("cannot get state commitment for block")
+		cache := &metrics.NoopCollector{}
+		commits := badger.NewCommits(cache, db)
+
+		stateCommitment, err = getStateCommitment(commits, blockID)
+		if err != nil {
+			log.Fatal().Err(err).Msg("cannot get state commitment for block")
+		}
+	}
+
+	if len(flagStateCommitment) > 0 {
+		var err error
+		stateCommitment, err = hex.DecodeString(flagStateCommitment)
+		if err != nil {
+			log.Fatal().Err(err).Msg("cannot get decode the state commitment")
+		}
 	}
 
 	log.Info().Msgf("Block state commitment: %s", hex.EncodeToString(stateCommitment))
 
-	err = extractExecutionState(flagExecutionStateDir, stateCommitment, flagOutputDir, log.Logger)
+	err := extractExecutionState(flagExecutionStateDir, stateCommitment, flagOutputDir, log.Logger)
 	if err != nil {
-		log.Fatal().Err(err).Msg("cannot generate checkpoint with state commitment")
+		log.Fatal().Err(err).Msgf("error extracting the execution state: %s", err.Error())
 	}
 }