feat: Update seq sender and aggregator (#38)

* feat: bring latest changes

aggregator/config.go

@@ -77,6 +77,12 @@ type Config struct {
 	// IntervalAfterWhichBatchConsolidateAnyway this is interval for the main sequencer, that will check if there is no transactions
 	IntervalAfterWhichBatchConsolidateAnyway types.Duration `mapstructure:"IntervalAfterWhichBatchConsolidateAnyway"`
 
+	// BatchProofSanityCheckEnabled is a flag to enable the sanity check of the batch proof
+	BatchProofSanityCheckEnabled bool `mapstructure:"BatchProofSanityCheckEnabled"`
+
+	// FinalProofSanityCheckEnabled is a flag to enable the sanity check of the final proof
+	FinalProofSanityCheckEnabled bool `mapstructure:"FinalProofSanityCheckEnabled"`
+
 	// ChainID is the L2 ChainID provided by the Network Config
 	ChainID uint64
 
@@ -142,6 +148,9 @@ type Config struct {
 
 	// AggLayerURL url of the agglayer service
 	AggLayerURL string `mapstructure:"AggLayerURL"`
+
+	// MaxWitnessRetrievalWorkers is the maximum number of workers that will be used to retrieve the witness
+	MaxWitnessRetrievalWorkers int `mapstructure:"MaxWitnessRetrievalWorkers"`
 }
 
 // StreamClientCfg contains the data streamer's configuration properties

aggregator/db/migrations/0002.sql

@@ -0,0 +1,8 @@
+-- +migrate Up
+DELETE FROM aggregator.batch;
+ALTER TABLE aggregator.batch
+    ADD COLUMN IF NOT EXISTS witness varchar NOT NULL;
+
+-- +migrate Down
+ALTER TABLE aggregator.batch
+    DROP COLUMN IF EXISTS witness;

aggregator/db/migrations/003.sql

@@ -0,0 +1,7 @@
+-- +migrate Up
+ALTER TABLE aggregator.batch
+    ALTER COLUMN witness DROP NOT NULL;
+
+-- +migrate Down
+ALTER TABLE aggregator.batch
+    ALTER COLUMN witness SET NOT NULL;

aggregator/interfaces.go

@@ -55,8 +55,8 @@ type stateInterface interface {
 	CleanupLockedProofs(ctx context.Context, duration string, dbTx pgx.Tx) (int64, error)
 	CheckProofExistsForBatch(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) (bool, error)
 	AddSequence(ctx context.Context, sequence state.Sequence, dbTx pgx.Tx) error
-	AddBatch(ctx context.Context, batch *state.Batch, datastream []byte, dbTx pgx.Tx) error
-	GetBatch(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) (*state.Batch, []byte, error)
+	AddBatch(ctx context.Context, dbBatch *state.DBBatch, dbTx pgx.Tx) error
+	GetBatch(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) (*state.DBBatch, error)
 	DeleteBatchesOlderThanBatchNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) error
 	DeleteBatchesNewerThanBatchNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) error
 }

aggregator/prover/prover.go

@@ -8,6 +8,7 @@ import (
 	"math/big"
 	"net"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/0xPolygon/cdk/config/types"
@@ -255,12 +256,19 @@ func (p *Prover) WaitRecursiveProof(ctx context.Context, proofID string) (string
 	if err != nil {
 		return "", common.Hash{}, err
 	}
-	stateRoot, err := GetStateRootFromProof(res.Proof.(*GetProofResponse_RecursiveProof).RecursiveProof)
-	if err != nil {
-		return "", common.Hash{}, err
-	}
+
 	resProof := res.Proof.(*GetProofResponse_RecursiveProof)
-	return resProof.RecursiveProof, stateRoot, nil
+
+	sr, err := GetStateRootFromProof(resProof.RecursiveProof)
+	if err != nil && sr != (common.Hash{}) {
+		log.Errorf("Error getting state root from proof: %v", err)
+	}
+
+	if sr == (common.Hash{}) {
+		log.Info("Recursive proof does not contain state root. Possibly mock prover is in use.")
+	}
+
+	return resProof.RecursiveProof, sr, nil
 }
 
 // WaitFinalProof waits for the final proof to be generated by the prover and
@@ -342,22 +350,27 @@ func (p *Prover) call(req *AggregatorMessage) (*ProverMessage, error) {
 
 // GetStateRootFromProof returns the state root from the proof.
 func GetStateRootFromProof(proof string) (common.Hash, error) {
+	// Log received proof
+	log.Debugf("Received proof to get SR from: %s", proof)
+
 	type Publics struct {
 		Publics []string `mapstructure:"publics"`
 	}
 
+	// Check if the proof contains the SR
+	if !strings.Contains(proof, "publics") {
+		return common.Hash{}, nil
+	}
+
 	var publics Publics
 	err := json.Unmarshal([]byte(proof), &publics)
 	if err != nil {
 		log.Errorf("Error unmarshalling proof: %v", err)
 		return common.Hash{}, err
 	}
 
-	var (
-		v [8]uint64
-		j = 0
-	)
-
+	var v [8]uint64
+	var j = 0
 	for i := stateRootStartIndex; i < stateRootFinalIndex; i++ {
 		u64, err := strconv.ParseInt(publics.Publics[i], 10, 64)
 		if err != nil {

config/default.go

@@ -57,6 +57,8 @@ ProofStatePollingInterval = "5s"
 SenderAddress = ""
 CleanupLockedProofsInterval = "2m"
 GeneratingProofCleanupThreshold = "10m"
+BatchProofSanityCheckEnabled = true
+FinalProofSanityCheckEnabled = true
 ForkId = 9
 GasOffset = 0
 WitnessURL = "localhost:8123"
@@ -65,6 +67,7 @@ UseFullWitness = false
 SettlementBackend = "l1"
 AggLayerTxTimeout = "5m"
 AggLayerURL = ""
+MaxWitnessRetrievalWorkers = 2
 SequencerPrivateKey = {}
 	[Aggregator.DB]
 		Name = "aggregator_db"

etherman/types.go

@@ -162,13 +162,14 @@ type Batch struct {
 }
 
 type SequenceBanana struct {
-	Batches              []Batch
-	OldAccInputHash      common.Hash
-	AccInputHash         common.Hash
-	L1InfoRoot           common.Hash
-	MaxSequenceTimestamp uint64
-	IndexL1InfoRoot      uint32
-	L2Coinbase           common.Address
+	Batches                []Batch
+	OldAccInputHash        common.Hash
+	AccInputHash           common.Hash
+	L1InfoRoot             common.Hash
+	MaxSequenceTimestamp   uint64
+	IndexL1InfoRoot        uint32
+	L2Coinbase             common.Address
+	LastVirtualBatchNumber uint64
 }
 
 func NewSequenceBanana(batches []Batch, l2Coinbase common.Address) *SequenceBanana {
@@ -198,3 +199,7 @@ func NewSequenceBanana(batches []Batch, l2Coinbase common.Address) *SequenceBana
 func (s *SequenceBanana) Len() int {
 	return len(s.Batches)
 }
+
+func (s *SequenceBanana) SetLastVirtualBatchNumber(batchNumber uint64) {
+	s.LastVirtualBatchNumber = batchNumber
+}

go.mod

@@ -6,8 +6,8 @@ require (
 	github.com/0xPolygon/cdk-contracts-tooling v0.0.0-20240726125827-301fa4c59245
 	github.com/0xPolygon/cdk-data-availability v0.0.8
 	github.com/0xPolygon/cdk-rpc v0.0.0-20240419104226-c0a62ba0f49d
-	github.com/0xPolygonHermez/zkevm-data-streamer v0.2.3-RC4
-	github.com/0xPolygonHermez/zkevm-ethtx-manager v0.1.9
+	github.com/0xPolygonHermez/zkevm-data-streamer v0.2.3
+	github.com/0xPolygonHermez/zkevm-ethtx-manager v0.1.10-0.20240716105056-c051c96d0234
 	github.com/0xPolygonHermez/zkevm-synchronizer-l1 v0.6.4
 	github.com/ethereum/go-ethereum v1.14.5
 	github.com/hermeznetwork/tracerr v0.3.2

go.sum

@@ -4,10 +4,10 @@ github.com/0xPolygon/cdk-data-availability v0.0.8 h1:bMmOYZ7Ei683y80ric3KzMPXtRG
 github.com/0xPolygon/cdk-data-availability v0.0.8/go.mod h1:3XkZ0zn0GsvAT01MPQMmukF534CVSFmtrcoK3F/BK6Q=
 github.com/0xPolygon/cdk-rpc v0.0.0-20240419104226-c0a62ba0f49d h1:sxh6hZ2jF/sxxj2jd5o1vuNNCZjYmn4aRG9SRlVaEFs=
 github.com/0xPolygon/cdk-rpc v0.0.0-20240419104226-c0a62ba0f49d/go.mod h1:2scWqMMufrQXu7TikDgQ3BsyaKoX8qP26D6E262vSOg=
-github.com/0xPolygonHermez/zkevm-data-streamer v0.2.3-RC4 h1:+4K+xSzv0ImbK30B/T9FauNTrTFUmWcNKYhIgwsE4C4=
-github.com/0xPolygonHermez/zkevm-data-streamer v0.2.3-RC4/go.mod h1:0QkAXcFa92mFJrCbN3UPUJGJYes851yEgYHLONnaosE=
-github.com/0xPolygonHermez/zkevm-ethtx-manager v0.1.9 h1:vrAezzwTNke6NroDAltGh1k2AJ6ibmZPBsG0bCltbRc=
-github.com/0xPolygonHermez/zkevm-ethtx-manager v0.1.9/go.mod h1:pRqfLQVM3nbzdhy3buqjAgcVyNDKAXOHqTSgkwiKpic=
+github.com/0xPolygonHermez/zkevm-data-streamer v0.2.3 h1:zJ06KCGLMDOap4slop/QmiMUO+VPsKSS3+944SY06ww=
+github.com/0xPolygonHermez/zkevm-data-streamer v0.2.3/go.mod h1:bv7DjATsczN2WvFt26jv34TWv6rfvYM1SqegrgrFwfI=
+github.com/0xPolygonHermez/zkevm-ethtx-manager v0.1.10-0.20240716105056-c051c96d0234 h1:QElCysO7f2xaknY/RDjxcs7IVmcgORfsCX2g+YD0Ko4=
+github.com/0xPolygonHermez/zkevm-ethtx-manager v0.1.10-0.20240716105056-c051c96d0234/go.mod h1:zBZWxwOHKlw+ghd9roQLgIkDZWA7e7qO3EsfQQT/+oQ=
 github.com/0xPolygonHermez/zkevm-synchronizer-l1 v0.6.3-0.20240712085301-0310358abb59 h1:Qwh92vFEXnpmDggQaZA3648viEQfLdMnAw/WFSY+2i8=
 github.com/0xPolygonHermez/zkevm-synchronizer-l1 v0.6.3-0.20240712085301-0310358abb59/go.mod h1:/LHf8jPQeBYKABM1xUmN1dKaFVIJc9jMQDSGBDJ7CS0=
 github.com/0xPolygonHermez/zkevm-synchronizer-l1 v0.6.3 h1:C+jNYr/CDMMn8wn3HqZqLTPU0luNYIB35pnxVf9O8TM=

sequencesender/config.go

@@ -64,6 +64,9 @@ type Config struct {
 
 	// MaxBatchesForL1 is the maximum amount of batches to be sequenced in a single L1 tx
 	MaxBatchesForL1 uint64 `mapstructure:"MaxBatchesForL1"`
+
+	// SanityCheckRPCURL is the URL of the RPC server to perform sanity check regarding the number of blocks in a batch
+	SanityCheckRPCURL string `mapstructure:"SanityCheckRPCURL"`
 }
 
 // StreamClientCfg contains the data streamer's configuration properties

sequencesender/seqsendertypes/types.go

@@ -34,11 +34,12 @@ type Sequence interface {
 	FirstBatch() Batch
 	LastBatch() Batch
 	Len() int
-
 	L2Coinbase() common.Address
+	LastVirtualBatchNumber() uint64
 
 	String() string
 	// WRITE
+	SetLastVirtualBatchNumber(batchNumber uint64)
 	//SetL1InfoRoot(hash common.Hash)
 	//SetOldAccInputHash(hash common.Hash)
 	//SetAccInputHash(hash common.Hash)