[E2E Test] Sequencer Hub disconnection and recovery

Makefile

@@ -218,6 +218,9 @@ e2e-test-admc-metadata-not-found-evm: clean-e2e
 e2e-test-update-do-timeout-unallowed-evm: clean-e2e
 	cd tests && go test -timeout=30m -race -v -run TestEIBCUpdateOnTimeout_Unallowed_EVM .
 
+e2e-test-sequencer-hub-disconnection-evm: clean-e2e
+	cd tests && go test -timeout=30m -race -v -run TestSequencerHubDisconnection_EVM .
+
 # Executes IBC tests via rollup-e2e-testing
 e2e-test-ibc-success-wasm: clean-e2e
 	cd tests && go test -timeout=30m -race -v -run TestIBCTransferSuccess_Wasm .

go.mod

@@ -40,7 +40,7 @@ require (
 	github.com/cosmos/gogoproto v1.4.10 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/deckarep/golang-set v1.8.0 // indirect
-	github.com/docker/docker v24.0.7+incompatible // indirect
+	github.com/docker/docker v24.0.7+incompatible
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect

tests/sequencer_test.go

@@ -0,0 +1,342 @@
+package tests
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zaptest"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/strslice"
+
+	test "github.com/decentrio/rollup-e2e-testing"
+	"github.com/decentrio/rollup-e2e-testing/cosmos/hub/celes_hub"
+	"github.com/decentrio/rollup-e2e-testing/cosmos/hub/dym_hub"
+	"github.com/decentrio/rollup-e2e-testing/cosmos/rollapp/dym_rollapp"
+	"github.com/decentrio/rollup-e2e-testing/relayer"
+
+	"github.com/decentrio/rollup-e2e-testing/ibc"
+	"github.com/decentrio/rollup-e2e-testing/testreporter"
+	"github.com/decentrio/rollup-e2e-testing/testutil"
+)
+
+func TestSequencerHubDisconnection_EVM(t *testing.T) {
+	if testing.Short() {
+		t.Skip()
+	}
+
+	ctx := context.Background()
+
+	// configFileOverrides := make(map[string]any)
+	dymintTomlOverrides := make(testutil.Toml)
+	dymintTomlOverrides["settlement_layer"] = "dymension"
+	dymintTomlOverrides["settlement_node_address"] = fmt.Sprintf("http://dymension_100-1-val-0-%s:26657", t.Name()) // Hub RPC not available
+	dymintTomlOverrides["rollapp_id"] = "rollappevm_1234-1"
+	dymintTomlOverrides["settlement_gas_prices"] = "0adym"
+	dymintTomlOverrides["max_idle_time"] = "3s"
+	dymintTomlOverrides["max_proof_time"] = "500ms"
+	dymintTomlOverrides["batch_submit_max_time"] = "80s"
+
+	configFileOverrides1 := make(map[string]any)
+	configTomlOverrides1 := make(testutil.Toml)
+	configTomlOverrides1["timeout_commit"] = "2s"
+	configTomlOverrides1["timeout_propose"] = "2s"
+	configTomlOverrides1["index_all_keys"] = "true"
+	configTomlOverrides1["mode"] = "validator"
+
+	configFileOverrides1["config/config.toml"] = configTomlOverrides1
+
+	// Create chain factory with dymension
+	numHubVals := 1
+	numHubFullNodes := 1
+	numRollAppFn := 0
+	numRollAppVals := 1
+	nodeStore := "/home/celestia/light"
+	p2pNetwork := "mocha-4"
+	coreIp := "celestia-testnet-consensus.itrocket.net"
+	// trustedHash := "\"017428B113893E854767E626BC9CF860BDF49C2AC2DF56F3C1B6582B2597AC6E\""
+	// sampleFrom := 2423882
+
+	url := "https://api-mocha.celenium.io/v1/block/count"
+	headerKey := "User-Agent"
+	headerValue := "Apidog/1.0.0 (https://apidog.com)"
+	rpcEndpoint := "http://celestia-testnet-consensus.itrocket.net:26657"
+
+	cf := test.NewBuiltinChainFactory(zaptest.NewLogger(t), []*test.ChainSpec{
+		{
+			Name: "celes-hub",
+			ChainConfig: ibc.ChainConfig{
+				Name:           "celestia",
+				Denom:          "utia",
+				Type:           "hub-celes",
+				GasPrices:      "0.002utia",
+				TrustingPeriod: "112h",
+				ChainID:        "test",
+				Bin:            "celestia-appd",
+				Images: []ibc.DockerImage{
+					{
+						Repository: "ghcr.io/decentrio/light",
+						Version:    "latest",
+						UidGid:     "1025:1025",
+					},
+				},
+				Bech32Prefix:        "celestia",
+				CoinType:            "118",
+				GasAdjustment:       1.5,
+				ConfigFileOverrides: configFileOverrides1,
+			},
+			NumValidators: &numHubVals,
+			NumFullNodes:  &numRollAppFn,
+		},
+	})
+
+	// Get chains from the chain factory
+	chains, err := cf.Chains(t.Name())
+	require.NoError(t, err)
+
+	celestia := chains[0].(*celes_hub.CelesHub)
+
+	// Relayer Factory
+	client, network := test.DockerSetup(t)
+
+	ic := test.NewSetup().
+		AddChain(celestia)
+
+	rep := testreporter.NewNopReporter()
+	eRep := rep.RelayerExecReporter(t)
+
+	err = ic.Build(ctx, eRep, test.InterchainBuildOptions{
+		TestName:         t.Name(),
+		Client:           client,
+		NetworkID:        network,
+		SkipPathCreation: false,
+
+		// This can be used to write to the block database which will index all block data e.g. txs, msgs, events, etc.
+		// BlockDatabaseFile: test.DefaultBlockDatabaseFilepath(),
+	}, nil, "", nil)
+	require.NoError(t, err)
+
+	validator, err := celestia.GetNode().AccountKeyBech32(ctx, "validator")
+	require.NoError(t, err)
+	// Get fund for submit blob
+	GetFaucet("http://18.184.170.181:3000/api/get-tia", validator)
+	err = testutil.WaitForBlocks(ctx, 8, celestia)
+	require.NoError(t, err)
+	GetFaucet("http://18.184.170.181:3000/api/get-tia", validator)
+	err = testutil.WaitForBlocks(ctx, 8, celestia)
+	require.NoError(t, err)
+	GetFaucet("http://18.184.170.181:3000/api/get-tia", validator)
+	err = testutil.WaitForBlocks(ctx, 8, celestia)
+	require.NoError(t, err)
+	GetFaucet("http://18.184.170.181:3000/api/get-tia", validator)
+	err = testutil.WaitForBlocks(ctx, 8, celestia)
+	require.NoError(t, err)
+	GetFaucet("http://18.184.170.181:3000/api/get-tia", validator)
+	err = testutil.WaitForBlocks(ctx, 8, celestia)
+	require.NoError(t, err)
+	GetFaucet("http://18.184.170.181:3000/api/get-tia", validator)
+	err = testutil.WaitForBlocks(ctx, 8, celestia)
+	require.NoError(t, err)
+	GetFaucet("http://18.184.170.181:3000/api/get-tia", validator)
+	err = testutil.WaitForBlocks(ctx, 8, celestia)
+	require.NoError(t, err)
+	GetFaucet("http://18.184.170.181:3000/api/get-tia", validator)
+	err = testutil.WaitForBlocks(ctx, 8, celestia)
+	require.NoError(t, err)
+	GetFaucet("http://18.184.170.181:3000/api/get-tia", validator)
+	err = testutil.WaitForBlocks(ctx, 8, celestia)
+	require.NoError(t, err)
+	GetFaucet("http://18.184.170.181:3000/api/get-tia", validator)
+	err = testutil.WaitForBlocks(ctx, 8, celestia)
+	require.NoError(t, err)
+	GetFaucet("http://18.184.170.181:3000/api/get-tia", validator)
+	err = testutil.WaitForBlocks(ctx, 8, celestia)
+	require.NoError(t, err)
+	GetFaucet("http://18.184.170.181:3000/api/get-tia", validator)
+	err = testutil.WaitForBlocks(ctx, 8, celestia)
+	require.NoError(t, err)
+	GetFaucet("http://18.184.170.181:3000/api/get-tia", validator)
+	err = testutil.WaitForBlocks(ctx, 8, celestia)
+	require.NoError(t, err)
+
+	err = celestia.GetNode().InitCelestiaDaLightNode(ctx, nodeStore, p2pNetwork, nil)
+	require.NoError(t, err)
+
+	err = testutil.WaitForBlocks(ctx, 3, celestia)
+	require.NoError(t, err)
+
+	// Change the file permissions
+	command := []string{"chmod", "-R", "777", "/home/celestia/light/config.toml"}
+
+	_, _, err = celestia.Exec(ctx, command, nil)
+	require.NoError(t, err)
+
+	file, err := os.Open("/tmp/celestia/light/config.toml")
+	require.NoError(t, err)
+	defer file.Close()
+
+	lastestBlockHeight, err := GetLatestBlockHeight(url, headerKey, headerValue)
+	require.NoError(t, err)
+	lastestBlockHeight = strings.TrimRight(lastestBlockHeight, "\n")
+	heightOfBlock, err := strconv.ParseInt(lastestBlockHeight, 10, 64) // base 10, bit size 64
+	require.NoError(t, err)
+
+	hash, err := celestia.GetNode().GetHashOfBlockHeightWithCustomizeRpcEndpoint(ctx, fmt.Sprintf("%d", heightOfBlock-2), rpcEndpoint)
+	require.NoError(t, err)
+
+	fmt.Println(hash)
+
+	hash = strings.TrimRight(hash, "\n")
+	var lines []string
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+	}
+
+	for i, line := range lines {
+		if strings.HasPrefix(line, "  TrustedHash =") {
+			lines[i] = fmt.Sprintf("  TrustedHash = \"%s\"", hash)
+		} else if strings.HasPrefix(line, "  SampleFrom =") {
+			lines[i] = fmt.Sprintf("  SampleFrom = %d", heightOfBlock-2)
+		} else if strings.HasPrefix(line, "  Address =") {
+			lines[i] = fmt.Sprintf("  Address = \"0.0.0.0\"")
+		}
+	}
+
+	output := strings.Join(lines, "\n")
+	file, err = os.Create("/tmp/celestia/light/config.toml")
+	require.NoError(t, err)
+	defer file.Close()
+
+	_, err = file.Write([]byte(output))
+	require.NoError(t, err)
+
+	containerID := fmt.Sprintf("test-val-0-%s", t.Name())
+
+	// Create an exec instance
+	execConfig := types.ExecConfig{
+		Cmd: strslice.StrSlice([]string{"celestia", "light", "start", "--node.store", nodeStore, "--gateway", "--core.ip", coreIp, "--p2p.network", p2pNetwork, "--keyring.accname", "validator"}), // Replace with your command and arguments
+	}
+
+	execIDResp, err := client.ContainerExecCreate(ctx, containerID, execConfig)
+	if err != nil {
+		panic(err)
+	}
+
+	execID := execIDResp.ID
+
+	// Start the exec instance
+	execStartCheck := types.ExecStartCheck{
+		Tty: false,
+	}
+
+	if err := client.ContainerExecStart(ctx, execID, execStartCheck); err != nil {
+		panic(err)
+	}
+
+	// _ = celestia.GetNode().StartCelestiaDaLightNode(ctx, nodeStore, coreIp, p2pNetwork, nil)
+	// require.NoError(t, err)
+
+	err = testutil.WaitForBlocks(ctx, 10, celestia)
+	require.NoError(t, err)
+
+	celestia_token, err := celestia.GetNode().GetAuthTokenCelestiaDaLight(ctx, p2pNetwork, nodeStore)
+	require.NoError(t, err)
+	println("check token: ", celestia_token)
+	celestia_namespace_id, err := RandomHex(10)
+	require.NoError(t, err)
+	println("check namespace: ", celestia_namespace_id)
+	da_config := fmt.Sprintf("{\"base_url\": \"http://test-val-0-%s:26658\", \"timeout\": 60000000000, \"gas_prices\":1.0, \"gas_adjustment\": 1.3, \"namespace_id\": \"%s\", \"auth_token\":\"%s\"}", t.Name(), celestia_namespace_id, celestia_token)
+
+	configFileOverrides := make(map[string]any)
+	dymintTomlOverrides["da_layer"] = "celestia"
+	dymintTomlOverrides["namespace_id"] = celestia_namespace_id
+	dymintTomlOverrides["da_config"] = da_config
+	configFileOverrides["config/dymint.toml"] = dymintTomlOverrides
+
+	cf = test.NewBuiltinChainFactory(zaptest.NewLogger(t), []*test.ChainSpec{
+		{
+			Name: "rollapp1",
+			ChainConfig: ibc.ChainConfig{
+				Type:                "rollapp-dym",
+				Name:                "rollapp-temp",
+				ChainID:             "rollappevm_1234-1",
+				Images:              []ibc.DockerImage{rollappEVMImage},
+				Bin:                 "rollappd",
+				Bech32Prefix:        "ethm",
+				Denom:               "urax",
+				CoinType:            "60",
+				GasPrices:           "0.0urax",
+				GasAdjustment:       1.1,
+				TrustingPeriod:      "112h",
+				EncodingConfig:      encodingConfig(),
+				NoHostMount:         false,
+				ModifyGenesis:       modifyRollappEVMGenesis(rollappEVMGenesisKV),
+				ConfigFileOverrides: configFileOverrides,
+			},
+			NumValidators: &numRollAppVals,
+			NumFullNodes:  &numRollAppFn,
+		},
+		{
+			Name:          "dymension-hub",
+			ChainConfig:   dymensionConfig,
+			NumValidators: &numHubVals,
+			NumFullNodes:  &numHubFullNodes,
+		},
+	})
+
+	// Get chains from the chain factory
+	chains, err = cf.Chains(t.Name())
+	require.NoError(t, err)
+
+	rollapp1 := chains[0].(*dym_rollapp.DymRollApp)
+	dymension := chains[1].(*dym_hub.DymHub)
+
+	r := test.NewBuiltinRelayerFactory(ibc.CosmosRly, zaptest.NewLogger(t),
+		relayer.CustomDockerImage(RelayerMainRepo, relayerVersion, "100:1000"), relayer.ImagePull(pullRelayerImage),
+	).Build(t, client, "relayer", network)
+
+	ic = test.NewSetup().
+		AddRollUp(dymension, rollapp1).
+		AddRelayer(r, "relayer").
+		AddLink(test.InterchainLink{
+			Chain1:  dymension,
+			Chain2:  rollapp1,
+			Relayer: r,
+			Path:    ibcPath,
+		})
+
+	err = ic.Build(ctx, eRep, test.InterchainBuildOptions{
+		TestName:         t.Name(),
+		Client:           client,
+		NetworkID:        network,
+		SkipPathCreation: true,
+
+		// This can be used to write to the block database which will index all block data e.g. txs, msgs, events, etc.
+		// BlockDatabaseFile: test.DefaultBlockDatabaseFilepath(),
+	}, nil, "", nil)
+	require.NoError(t, err)
+
+	dymension.StopAllNodes(ctx)
+
+	rollappHeight, err := rollapp1.GetNode().Height(ctx)
+	require.NoError(t, err)
+
+	// wait until the packet is finalized
+	isFinalized, err := dymension.WaitUntilRollappHeightIsFinalized(ctx, rollapp1.GetChainID(), rollappHeight, 180)
+	require.Error(t, err)
+	require.False(t, isFinalized)
+
+	dymension.StartAllNodes(ctx)
+
+	// wait until the packet is finalized
+	isFinalized, err = dymension.WaitUntilRollappHeightIsFinalized(ctx, rollapp1.GetChainID(), rollappHeight, 300)
+	require.NoError(t, err)
+	require.True(t, isFinalized)
+}