Extend Rosetta to work with MyCelo testnets

README.md

@@ -209,6 +209,19 @@ go run examples/generate_balances/main.go \
   rosetta-cli-conf/testnet/bootstrap_balances.json
 ```
 
+### Running Rosetta with a mycelo testnet
+
+- Set `--geth.genesis` to point to the genesis file for the testnet.
+- Set `--geth.networkid` to the network ID (if this is a deployed testnet, this may be different from the `ChainID` in the genesis file). If this value is the same as the `ChainID`, it is not necessary to set this parameter.
+- Set the `--monitor.initcontracts` flag (at least on the first run), which fetches necessary state from the genesis block and updates the Rosetta DB accordingly.
+
+To run reconciliation tests on this network:
+
+- Generate `bootstrap_balances.json` using the network's genesis block.
+- In the `cli-config.json`:
+  - Set `network` to match the `ChainID` in the genesis file (not the network ID).
+  - Point `bootstrap_balances` to the generated `bootstrap_balances.json`.
+
 ## Releasing rosetta
 
 ### Versioning

cmd/run.go

@@ -79,6 +79,7 @@ func init() {
 
 	flagSet.String("geth.genesis", "", "(Optional) path to the genesis.json, for use with custom chains")
 	utils.ExitOnError(serveCmd.MarkFlagFilename("geth.genesis", "json"))
+	flagSet.String("geth.networkid", "", "(Optional) Network ID, for use with custom chains")
 	// Note that we do not set any default here because it would clash with geth.genesis if that was defined.
 	flagSet.String("geth.network", "", "Network to use, either 'mainnet', 'alfajores', or 'baklava'")
 
@@ -96,6 +97,9 @@ func init() {
 	flagSet.String("geth.syncmode", "fast", "Geth blockchain sync mode (fast, full, light)")
 	flagSet.String("geth.gcmode", "full", "Geth garbage collection mode (full, archive)")
 	flagSet.String("geth.maxpeers", "1100", "Maximum number of network peers (network disabled if set to 0)")
+
+	// Monitor Service Flags
+	flagSet.Bool("monitor.initcontracts", false, "Set to true to properly initialize contract state, i.e. when running MyCelo testnets")
 }
 
 func getDatadir(cmd *cobra.Command) string {
@@ -114,6 +118,7 @@ func readGethOption(cmd *cobra.Command, datadir string) *geth.GethOpts {
 		GethBinary:  viper.GetString("geth.binary"),
 		GenesisPath: viper.GetString("geth.genesis"),
 		Network:     viper.GetString("geth.network"),
+		NetworkId:   viper.GetString("geth.networkid"),
 		Datadir:     filepath.Join(datadir, "celo"),
 		LogsPath:    viper.GetString("geth.logfile"),
 		IpcPath:     viper.GetString("geth.ipcpath"),
@@ -138,6 +143,8 @@ func readGethOption(cmd *cobra.Command, datadir string) *geth.GethOpts {
 		printUsageAndExit(cmd, "Missing config option for 'geth.genesis' or 'geth.network'")
 	} else if opts.GenesisPath != "" && opts.Network != "" {
 		printUsageAndExit(cmd, "Must provide exactly one of 'geth.genesis' or 'geth.network'")
+	} else if opts.NetworkId != "" && opts.GenesisPath == "" {
+		printUsageAndExit(cmd, "Must provide 'geth.genesis' when using 'geth.networkid'")
 	}
 
 	return opts
@@ -238,8 +245,14 @@ loop:
 		}
 		return nil
 	})
+
 	grp.Go(func() error {
-		err = monitor.NewMonitorService(cc, celoStore, chainParams.IsGingerbread).Start(ctx)
+		err = monitor.NewMonitorService(
+			cc,
+			celoStore,
+			chainParams.IsGingerbread,
+			viper.GetBool("monitor.initcontracts"),
+		).Start(ctx)
 		if err != nil {
 			fmt.Println("error running mon serrvice")
 			ec.Add(fmt.Errorf("error running monitor service : %w", err))

service/geth/geth.go

@@ -36,6 +36,7 @@ type GethOpts struct {
 	GethBinary  string
 	GenesisPath string
 	Network     string
+	NetworkId   string
 	IpcPath     string
 	LogsPath    string
 	Datadir     string
@@ -236,7 +237,15 @@ func (gs *gethService) startGeth(stdErr *os.File) error {
 	case params.MainnetChainConfig.ChainID.String():
 		break
 	default:
-		gethArgs = append(gethArgs, "--networkid", gs.chainParams.ChainId.String())
+		gs.logger.Info("Setting networkId")
+		var networkId string
+		if gs.opts.NetworkId != "" {
+			networkId = gs.opts.NetworkId
+		} else {
+			gs.logger.Info("Using genesis ChainId as NetworkId")
+			networkId = gs.chainParams.ChainId.String()
+		}
+		gethArgs = append(gethArgs, "--networkid", networkId)
 	}
 
 	if gs.opts.Verbosity != "" {

service/monitor/service.go

@@ -33,16 +33,18 @@ type monitorService struct {
 	db            db.RosettaDB
 	logger        log.Logger
 	isGingerbread func(*big.Int) bool
+	initContracts bool // Necessary for running with mycelo testnets
 }
 
 const srvName = "celo-monitor"
 
-func NewMonitorService(cc *client.CeloClient, db db.RosettaDB, isGingerbread func(*big.Int) bool) *monitorService {
+func NewMonitorService(cc *client.CeloClient, db db.RosettaDB, isGingerbread func(*big.Int) bool, initContracts bool) *monitorService {
 	return &monitorService{
 		cc:            cc,
 		db:            db,
 		logger:        log.New("srv", srvName),
 		isGingerbread: isGingerbread,
+		initContracts: initContracts,
 	}
 }
 
@@ -70,6 +72,12 @@ func (ms *monitorService) Start(ctx context.Context) error {
 		return err
 	}
 
+	if ms.initContracts && startBlock.Cmp(big.NewInt(1)) < 0 {
+		if err := UpdateDBForBlock(ctx, startBlock, ms.cc, ms.db, ms.logger); err != nil {
+			return err
+		}
+	}
+
 	ms.logger.Info("Resuming operation from last persisted  block", "block", startBlock)
 	startBlock = utils.Inc(startBlock)
 

service/monitor/static_processor.go

@@ -0,0 +1,144 @@
+// Copyright 2023 Celo Org
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package monitor
+
+import (
+	"context"
+	"math/big"
+
+	"github.com/celo-org/celo-blockchain/accounts/abi/bind"
+	"github.com/celo-org/celo-blockchain/common"
+	"github.com/celo-org/celo-blockchain/log"
+	"github.com/celo-org/kliento/client"
+	"github.com/celo-org/kliento/contracts"
+	"github.com/celo-org/kliento/registry"
+	"github.com/celo-org/rosetta/db"
+)
+
+type staticProcessor struct {
+	callOpts      *bind.CallOpts
+	boundRegistry *contracts.Registry
+	cc            *client.CeloClient
+	logger        log.Logger
+}
+
+func newStaticProcessor(
+	ctx context.Context,
+	blockNumber *big.Int,
+	cc *client.CeloClient,
+	logger log.Logger,
+) (*staticProcessor, error) {
+	boundRegistry, err := contracts.NewRegistry(registry.RegistryAddress, cc.Eth)
+	if err != nil {
+		return nil, err
+	}
+	callOpts := &bind.CallOpts{
+		Context:     ctx,
+		BlockNumber: blockNumber,
+	}
+	return &staticProcessor{
+		callOpts:      callOpts,
+		boundRegistry: boundRegistry,
+		logger:        logger.New("pipe", "static_processor"),
+		cc:            cc,
+	}, nil
+}
+
+func (sf *staticProcessor) fetchContractAddresses() ([]db.RegistryChange, error) {
+	contractAddresses := []db.RegistryChange{}
+
+	for _, contractId := range registry.RegisteredContractIDs {
+		contractStrName := contractId.String()
+		contractAddress, err := sf.boundRegistry.GetAddressForString(sf.callOpts, contractStrName)
+		if err != nil {
+			return nil, err
+		}
+		contractAddresses = append(contractAddresses, db.RegistryChange{
+			TxIndex:    0,
+			Contract:   contractStrName,
+			NewAddress: contractAddress,
+		})
+		sf.logger.Info("Found Core Contract address", "name", contractStrName, "newAddress", contractAddress.Hex())
+	}
+	return contractAddresses, nil
+}
+
+func (sf *staticProcessor) fetchGasPriceMinimum() (*big.Int, error) {
+	gpmAddress, err := sf.boundRegistry.GetAddressForString(sf.callOpts, registry.GasPriceMinimumContractID.String())
+	if err != nil || gpmAddress == common.ZeroAddress {
+		return nil, err
+	}
+	gpmContract, err := contracts.NewGasPriceMinimum(gpmAddress, sf.cc.Eth)
+	if err != nil {
+		return nil, err
+	}
+	return gpmContract.GetGasPriceMinimum(sf.callOpts, common.ZeroAddress)
+}
+
+func (sf *staticProcessor) fetchCarbonOffsetPartner() (*db.CarbonOffsetPartnerChange, error) {
+	epochRewardsAddress, err := sf.boundRegistry.GetAddressForString(sf.callOpts, registry.EpochRewardsContractID.String())
+	if err != nil || epochRewardsAddress == common.ZeroAddress {
+		return nil, err
+	}
+	epochRewards, err := contracts.NewEpochRewards(epochRewardsAddress, sf.cc.Eth)
+	if err != nil {
+		return nil, err
+	}
+	offsettingPartner, err := epochRewards.CarbonOffsettingPartner(sf.callOpts)
+	if err != nil {
+		return nil, err
+	}
+	return &db.CarbonOffsetPartnerChange{TxIndex: 0, Address: offsettingPartner}, nil
+}
+
+// Fetches relevant state from block and updates the Rosetta DB accordingly.
+// This is necessary for running Rosetta on networks that have Core Contracts
+// already deployed at genesis (e.g. myCelo networks).
+func UpdateDBForBlock(
+	ctx context.Context,
+	blockNumber *big.Int,
+	cc *client.CeloClient,
+	rosettaDB db.RosettaDB,
+	logger log.Logger,
+) error {
+	sf, err := newStaticProcessor(ctx, blockNumber, cc, logger)
+	if err != nil {
+		return err
+	}
+	sf.logger.Info("Fetching contract state", "block", blockNumber)
+	contractAddresses, err := sf.fetchContractAddresses()
+	if err != nil {
+		return err
+	}
+	gpm, err := sf.fetchGasPriceMinimum()
+	if err != nil {
+		return err
+	}
+	sf.logger.Info("Found GasPriceMinimum", "block", blockNumber, "GPM", gpm)
+	carbonOffsetPartner, err := sf.fetchCarbonOffsetPartner()
+	if err != nil {
+		return err
+	}
+	if carbonOffsetPartner != nil {
+		sf.logger.Info("Found CarbonOffsetPartner", "block", blockNumber, "address", carbonOffsetPartner.Address)
+	}
+
+	return rosettaDB.ApplyChanges(ctx, &db.BlockChangeSet{
+		BlockNumber:               blockNumber,
+		RegistryChanges:           contractAddresses,
+		GasPriceMinimum:           gpm,
+		CarbonOffsetPartnerChange: *carbonOffsetPartner,
+	})
+}