Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat!(sidecar): add start check callback with rollkit example #1118

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion chain/cosmos/chain_node.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ func (tn *ChainNode) NewClient(addr string) error {
func (tn *ChainNode) NewSidecarProcess(
ctx context.Context,
preStart bool,
startCheck func(int) error,
processName string,
cli *dockerclient.Client,
networkID string,
Expand All @@ -160,7 +161,7 @@ func (tn *ChainNode) NewSidecarProcess(
startCmd []string,
env []string,
) error {
s := NewSidecar(tn.log, true, preStart, tn.Chain, cli, networkID, processName, tn.TestName, image, homeDir, tn.Index, ports, startCmd, env)
s := NewSidecar(tn.log, true, preStart, startCheck, tn.Chain, cli, networkID, processName, tn.TestName, image, homeDir, tn.Index, ports, startCmd, env)

v, err := cli.VolumeCreate(ctx, volumetypes.CreateOptions{
Labels: map[string]string{
Expand Down
11 changes: 8 additions & 3 deletions chain/cosmos/cosmos_chain.go
Original file line number Diff line number Diff line change
Expand Up @@ -653,7 +653,7 @@ func (c *CosmosChain) NewChainNode(
continue
}

err = tn.NewSidecarProcess(ctx, cfg.PreStart, cfg.ProcessName, cli, networkID, cfg.Image, cfg.HomeDir, cfg.Ports, cfg.StartCmd, cfg.Env)
err = tn.NewSidecarProcess(ctx, cfg.PreStart, cfg.StartCheck, cfg.ProcessName, cli, networkID, cfg.Image, cfg.HomeDir, cfg.Ports, cfg.StartCmd, cfg.Env)
if err != nil {
return nil, err
}
Expand All @@ -666,6 +666,7 @@ func (c *CosmosChain) NewChainNode(
func (c *CosmosChain) NewSidecarProcess(
ctx context.Context,
preStart bool,
startCheck func(int) error,
processName string,
testName string,
cli *client.Client,
Expand All @@ -679,7 +680,7 @@ func (c *CosmosChain) NewSidecarProcess(
) error {
// Construct the SidecarProcess first so we can access its name.
// The SidecarProcess's VolumeName cannot be set until after we create the volume.
s := NewSidecar(c.log, false, preStart, c, cli, networkID, processName, testName, image, homeDir, index, ports, startCmd, env)
s := NewSidecar(c.log, false, preStart, startCheck, c, cli, networkID, processName, testName, image, homeDir, index, ports, startCmd, env)

v, err := cli.VolumeCreate(ctx, volumetypes.CreateOptions{
Labels: map[string]string{
Expand Down Expand Up @@ -776,7 +777,7 @@ func (c *CosmosChain) initializeSidecars(
}

eg.Go(func() error {
err := c.NewSidecarProcess(egCtx, cfg.PreStart, cfg.ProcessName, testName, cli, networkID, cfg.Image, cfg.HomeDir, i, cfg.Ports, cfg.StartCmd, cfg.Env)
err := c.NewSidecarProcess(egCtx, cfg.PreStart, cfg.StartCheck, cfg.ProcessName, testName, cli, networkID, cfg.Image, cfg.HomeDir, i, cfg.Ports, cfg.StartCmd, cfg.Env)
if err != nil {
return err
}
Expand Down Expand Up @@ -994,6 +995,10 @@ func (c *CosmosChain) Start(testName string, ctx context.Context, additionalGene
return err
}

if s.startCheck != nil {
return s.startCheck(s.Index)
}

return nil
})
}
Expand Down
5 changes: 4 additions & 1 deletion chain/cosmos/sidecar.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ type SidecarProcess struct {
validatorProcess bool

// If true this process should be started before the chain or validator, otherwise it should be explicitly started after.
preStart bool
preStart bool
startCheck func(int) error

ProcessName string
TestName string
Expand All @@ -48,6 +49,7 @@ func NewSidecar(
log *zap.Logger,
validatorProcess bool,
preStart bool,
startCheck func(int) error,
chain ibc.Chain,
dockerClient *dockerclient.Client,
networkID, processName, testName string,
Expand All @@ -73,6 +75,7 @@ func NewSidecar(
Index: index,
Chain: chain,
preStart: preStart,
startCheck: startCheck,
validatorProcess: validatorProcess,
ProcessName: processName,
TestName: testName,
Expand Down
221 changes: 221 additions & 0 deletions examples/cosmos/rollkit_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,221 @@
package cosmos_test

import (
"context"
"cosmossdk.io/math"
"encoding/base64"
"fmt"
"github.com/avast/retry-go/v4"
cmtjson "github.com/cometbft/cometbft/libs/json"
"github.com/cometbft/cometbft/privval"
coretypes "github.com/cometbft/cometbft/rpc/core/types"
sdk "github.com/cosmos/cosmos-sdk/types"
"github.com/strangelove-ventures/interchaintest/v8"
"github.com/strangelove-ventures/interchaintest/v8/chain/cosmos"
"github.com/strangelove-ventures/interchaintest/v8/ibc"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
"strings"
"testing"
"time"
)

var sdk50Genesis = []cosmos.GenesisKV{
{
Key: "app_state.gov.params.voting_period",
Value: votingPeriod,
},
{
Key: "app_state.gov.params.max_deposit_period",
Value: maxDepositPeriod,
},
{
Key: "app_state.gov.params.min_deposit.0.denom",
Value: "stake",
},
{
Key: "app_state.gov.params.min_deposit.0.amount",
Value: "1",
},
}

func TestRollkitCelestiaDevnet(t *testing.T) {
ctx := context.Background()
var rollkitChain *cosmos.CosmosChain

cf := interchaintest.NewBuiltinChainFactory(zaptest.NewLogger(t), []*interchaintest.ChainSpec{
{
Name: "gm",
ChainName: "gm",
Version: "tutorial-local-da",
ChainConfig: ibc.ChainConfig{
Type: "cosmos",
Name: "gm",
ChainID: "gm",
Images: []ibc.DockerImage{
{
Repository: "ghcr.io/gjermundgaraba/gm",
Copy link
Member

@Reecepbcups Reecepbcups May 10, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this just the standard https://github.com/rollkit/gm or are there changes made to work with this?

Ref https://github.com/rollchains/tiablob/blob/main/interchaintest/setup_celestia.go as well vs using a combined instance. I should probably get this moved here and help clean up some setup

Copy link
Contributor Author

@gjermundgaraba gjermundgaraba May 10, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just standard, but there is no updated docker builds (and the main branch is outdated) so I just took the tutorial-local-da branch and built a docker image of that

Version: "tutorial-local-da",
UidGid: "10001:10001",
},
},
Bin: "gmd",
Bech32Prefix: "gm",
Denom: "stake",
CoinType: "118",
SigningAlgorithm: "",
GasPrices: "0stake",
GasAdjustment: 2.0,
TrustingPeriod: "112h",
NoHostMount: false,
SkipGenTx: false,
PreGenesis: nil,
ModifyGenesis: func(config ibc.ChainConfig, bytes []byte) ([]byte, error) {
Copy link
Member

@Reecepbcups Reecepbcups May 10, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ahh I see now. Ideally we would abstract this away and have a RollKit section which does this but does not have the user be informed of all the setup complexities. Everything here looks reasonable imo

great work!

I need to think on it for how we can handle. Maybe a IsRollkitChain: true (or if we can somehow figure this out without user interaction, i.e. something within the filesystem or part of the main binary cobra)

then this would all run after modify genesis with a celestia-da

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, it would be great to automagically sort this somehow without all the manual stuff for sure :)

valKeyFileBz, _, err := rollkitChain.Validators[0].Exec(ctx, []string{"cat", "/var/cosmos-chain/gm/config/priv_validator_key.json"}, []string{})
if err != nil {
return nil, err
}
var pvKey privval.FilePVKey
if err = cmtjson.Unmarshal(valKeyFileBz, &pvKey); err != nil {
return nil, err
}

newGenesis := append(sdk50Genesis, cosmos.GenesisKV{
Key: "consensus.validators",
Value: []map[string]interface{}{
{
"address": pvKey.Address.String(),
"pub_key": map[string]interface{}{
"type": "tendermint/PubKeyEd25519",
"value": base64.StdEncoding.EncodeToString(pvKey.PubKey.Bytes()),
},
"power": "1000", // This is important
"name": "Rollkit Sequencer", // This is important
},
},
})

daHostName := rollkitChain.Sidecars[0].HostName()

if _, _, err := rollkitChain.Sidecars[0].Exec(ctx, []string{"mkdir", "/home/celestia/bridge"}, []string{}); err != nil {
return nil, err
}

daAuthTokenBz, _, err := rollkitChain.Sidecars[0].Exec(ctx, []string{"celestia", "bridge", "auth", "admin", "--node.store", "/home/celestia/bridge"}, []string{})
if err != nil {
return nil, err
}
daAuthToken := strings.TrimSuffix(string(daAuthTokenBz), "\n")

if _, _, err := rollkitChain.Validators[0].Exec(ctx, []string{"bash", "-c", fmt.Sprintf(`echo "[rollkit]
da_address = \"http://%s:%s\"
da_auth_token = \"%s\"" >> /var/cosmos-chain/gm/config/config.toml`, daHostName, "26658", daAuthToken)}, []string{}); err != nil {
return nil, err
}

return cosmos.ModifyGenesis(newGenesis)(config, bytes)
},
ModifyGenesisAmounts: func(_ int) (sdk.Coin, sdk.Coin) {
return sdk.NewInt64Coin("stake", 10_000_000_000_000), sdk.NewInt64Coin("stake", 1_000_000_000)
},
ConfigFileOverrides: nil,
EncodingConfig: nil,
UsingChainIDFlagCLI: false,
SidecarConfigs: []ibc.SidecarConfig{
{
ProcessName: "local-celestia-devnet",
Image: ibc.DockerImage{
Repository: "ghcr.io/rollkit/local-celestia-devnet",
Version: "latest",
UidGid: "1025:1025",
},
HomeDir: "/home/celestia",
Ports: []string{
"26650/tcp",
"26657/tcp",
"26658/tcp",
"26659/tcp",
"9090/tcp",
},
StartCmd: []string{"/bin/bash", "/opt/entrypoint.sh"},
Env: nil, // Here we could set CELESTIA_NAMESPACE if needed
PreStart: true,
StartCheck: func(index int) error {
Copy link
Member

@Reecepbcups Reecepbcups May 10, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

note to self: Maybe to be more inline with #1123, we rename this to PostStart() on the sidecar process. Just so it more closely matches that PRs API conventions and also is more versatile vs just a standard check.

Does not seem the index here is used? Could put the CosmosChain here or something

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That naming makes sense.

Yeah, the idea was originally to send in the SidecarProcess but that gets to a cyclical dependency so ended up with an index and then in the end didn't use it.

In the code below I just do Sidecars[0], but I guess the idea was to access the right one. But I suppose I would always know the order they are added (being the same as the SidecarConfigs slice order). So could send in the chain like you said instead.

return retry.Do(func() error {
daHostName := rollkitChain.Sidecars[0].HostName()
_, errOut, err := rollkitChain.Sidecars[0].Exec(ctx, []string{"celestia-appd", "status", "--node", fmt.Sprintf("http://%s:26657", daHostName)}, []string{})
if err != nil {
return err
}

var status coretypes.ResultStatus
if err = cmtjson.Unmarshal(errOut, &status); err != nil {
return err
}

if status.SyncInfo.CatchingUp {
return fmt.Errorf("node is still catching up")
}

time.Sleep(5 * time.Second) // just for good measure

return nil
}, retry.Context(ctx), retry.Attempts(40), retry.Delay(3*time.Second), retry.DelayType(retry.FixedDelay))
},
ValidatorProcess: false,
},
},
AdditionalStartArgs: []string{"--rollkit.aggregator", "true", "--rollkit.da_start_height", "1", "--api.enable", "--api.enabled-unsafe-cors"},
},
NumValidators: &numValsOne,
NumFullNodes: &numFullNodesZero,
},
})

chains, err := cf.Chains(t.Name())
require.NoError(t, err)

rollkitChain = chains[0].(*cosmos.CosmosChain)

ic := interchaintest.NewInterchain().
AddChain(rollkitChain)

client, network := interchaintest.DockerSetup(t)

require.NoError(t, ic.Build(ctx, nil, interchaintest.InterchainBuildOptions{
TestName: t.Name(),
Client: client,
NetworkID: network,
SkipPathCreation: true,
}))
t.Cleanup(func() {
_ = ic.Close()
})

// Faucet funds to a user
users := interchaintest.GetAndFundTestUsers(t, ctx, "default", genesisFundsAmt, rollkitChain, rollkitChain)
user := users[0]
user2 := users[1]

// get the users balance
initBal, err := rollkitChain.GetBalance(ctx, user.FormattedAddress(), "stake")
require.NoError(t, err)

// Send many transactions in a row
for i := 0; i < 10; i++ {
require.NoError(t, rollkitChain.SendFunds(ctx, user.KeyName(), ibc.WalletAmount{
Address: user2.FormattedAddress(),
Denom: "stake",
Amount: math.NewInt(1),
}))
require.NoError(t, rollkitChain.SendFunds(ctx, user2.KeyName(), ibc.WalletAmount{
Address: user.FormattedAddress(),
Denom: "stake",
Amount: math.NewInt(1),
}))
}

endBal, err := rollkitChain.GetBalance(ctx, user.FormattedAddress(), "stake")
require.NoError(t, err)
require.EqualValues(t, initBal, endBal)
}
1 change: 1 addition & 0 deletions ibc/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,7 @@ type SidecarConfig struct {
StartCmd []string
Env []string
PreStart bool
StartCheck func(int) error
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a good addition

ValidatorProcess bool
}

Expand Down
Loading