diff --git a/chain/cosmos/chain_node.go b/chain/cosmos/chain_node.go index 591e8d3eb..1d326cc3c 100644 --- a/chain/cosmos/chain_node.go +++ b/chain/cosmos/chain_node.go @@ -23,6 +23,7 @@ import ( authTx "github.com/cosmos/cosmos-sdk/x/auth/tx" paramsutils "github.com/cosmos/cosmos-sdk/x/params/client/utils" stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" + volumetypes "github.com/docker/docker/api/types/volume" dockerclient "github.com/docker/docker/client" "github.com/docker/go-connections/nat" "github.com/strangelove-ventures/interchaintest/v6/ibc" @@ -51,6 +52,9 @@ type ChainNode struct { TestName string Image ibc.DockerImage + // Additional processes that need to be run on a per-validator basis. + Sidecars SidecarProcesses + lock sync.Mutex log *zap.Logger @@ -121,6 +125,48 @@ func (tn *ChainNode) NewClient(addr string) error { return nil } +func (tn *ChainNode) NewSidecarProcess( + ctx context.Context, + preStart bool, + processName string, + cli *dockerclient.Client, + networkID string, + image ibc.DockerImage, + homeDir string, + ports []string, + startCmd []string, +) error { + s := NewSidecar(tn.log, true, preStart, tn.Chain, cli, networkID, processName, tn.TestName, image, homeDir, tn.Index, ports, startCmd) + + v, err := cli.VolumeCreate(ctx, volumetypes.CreateOptions{ + Labels: map[string]string{ + dockerutil.CleanupLabel: tn.TestName, + dockerutil.NodeOwnerLabel: s.Name(), + }, + }) + if err != nil { + return fmt.Errorf("creating volume for sidecar process: %w", err) + } + s.VolumeName = v.Name + + if err := dockerutil.SetVolumeOwner(ctx, dockerutil.VolumeOwnerOptions{ + Log: tn.log, + + Client: cli, + + VolumeName: v.Name, + ImageRef: image.Ref(), + TestName: tn.TestName, + UidGid: image.UidGid, + }); err != nil { + return fmt.Errorf("set volume owner: %w", err) + } + + tn.Sidecars = append(tn.Sidecars, s) + + return nil +} + // CliContext creates a new Cosmos SDK client context func (tn *ChainNode) CliContext() client.Context { cfg := tn.Chain.Config() @@ -911,6 +957,20 @@ func (tn *ChainNode) CreateNodeContainer(ctx context.Context) error { } func (tn *ChainNode) StartContainer(ctx context.Context) error { + for _, s := range tn.Sidecars { + err := s.containerLifecycle.Running(ctx) + + if s.preStart && err != nil { + if err := s.CreateContainer(ctx); err != nil { + return err + } + + if err := s.StartContainer(ctx); err != nil { + return err + } + } + } + if err := tn.containerLifecycle.StartContainer(ctx); err != nil { return err } @@ -943,10 +1003,20 @@ func (tn *ChainNode) StartContainer(ctx context.Context) error { } func (tn *ChainNode) StopContainer(ctx context.Context) error { + for _, s := range tn.Sidecars { + if err := s.StopContainer(ctx); err != nil { + return err + } + } return tn.containerLifecycle.StopContainer(ctx) } func (tn *ChainNode) RemoveContainer(ctx context.Context) error { + for _, s := range tn.Sidecars { + if err := s.RemoveContainer(ctx); err != nil { + return err + } + } return tn.containerLifecycle.RemoveContainer(ctx) } diff --git a/chain/cosmos/cosmos_chain.go b/chain/cosmos/cosmos_chain.go index f15d1c734..0ae3a5083 100644 --- a/chain/cosmos/cosmos_chain.go +++ b/chain/cosmos/cosmos_chain.go @@ -43,6 +43,9 @@ type CosmosChain struct { Validators ChainNodes FullNodes ChainNodes + // Additional processes that need to be run on a per-chain basis. + Sidecars SidecarProcesses + log *zap.Logger keyring keyring.Keyring findTxMu sync.Mutex @@ -165,6 +168,9 @@ func (c *CosmosChain) Config() ibc.ChainConfig { // Implements Chain interface func (c *CosmosChain) Initialize(ctx context.Context, testName string, cli *client.Client, networkID string) error { + if err := c.initializeSidecars(ctx, testName, cli, networkID); err != nil { + return err + } return c.initializeChainNodes(ctx, testName, cli, networkID) } @@ -543,9 +549,68 @@ func (c *CosmosChain) NewChainNode( }); err != nil { return nil, fmt.Errorf("set volume owner: %w", err) } + + for _, cfg := range c.cfg.SidecarConfigs { + if !cfg.ValidatorProcess { + continue + } + + err = tn.NewSidecarProcess(ctx, cfg.PreStart, cfg.ProcessName, cli, networkID, cfg.Image, cfg.HomeDir, cfg.Ports, cfg.StartCmd) + if err != nil { + return nil, err + } + } + return tn, nil } +// NewSidecarProcess constructs a new sidecar process with a docker volume. +func (c *CosmosChain) NewSidecarProcess( + ctx context.Context, + preStart bool, + processName string, + testName string, + cli *client.Client, + networkID string, + image ibc.DockerImage, + homeDir string, + index int, + ports []string, + startCmd []string, +) error { + // Construct the SidecarProcess first so we can access its name. + // The SidecarProcess's VolumeName cannot be set until after we create the volume. + s := NewSidecar(c.log, false, preStart, c, cli, networkID, processName, testName, image, homeDir, index, ports, startCmd) + + v, err := cli.VolumeCreate(ctx, volumetypes.CreateOptions{ + Labels: map[string]string{ + dockerutil.CleanupLabel: testName, + dockerutil.NodeOwnerLabel: s.Name(), + }, + }) + if err != nil { + return fmt.Errorf("creating volume for sidecar process: %w", err) + } + s.VolumeName = v.Name + + if err := dockerutil.SetVolumeOwner(ctx, dockerutil.VolumeOwnerOptions{ + Log: c.log, + + Client: cli, + + VolumeName: v.Name, + ImageRef: image.Ref(), + TestName: testName, + UidGid: image.UidGid, + }); err != nil { + return fmt.Errorf("set volume owner: %w", err) + } + + c.Sidecars = append(c.Sidecars, s) + + return nil +} + // creates the test node objects required for bootstrapping tests func (c *CosmosChain) initializeChainNodes( ctx context.Context, @@ -595,6 +660,37 @@ func (c *CosmosChain) initializeChainNodes( return nil } +// initializeSidecars creates the sidecar processes that exist at the chain level. +func (c *CosmosChain) initializeSidecars( + ctx context.Context, + testName string, + cli *client.Client, + networkID string, +) error { + eg, egCtx := errgroup.WithContext(ctx) + for i, cfg := range c.cfg.SidecarConfigs { + i := i + cfg := cfg + + if cfg.ValidatorProcess { + continue + } + + eg.Go(func() error { + err := c.NewSidecarProcess(egCtx, cfg.PreStart, cfg.ProcessName, testName, cli, networkID, cfg.Image, cfg.HomeDir, i, cfg.Ports, cfg.StartCmd) + if err != nil { + return err + } + return nil + }) + + } + if err := eg.Wait(); err != nil { + return err + } + return nil +} + type GenesisValidatorPubKey struct { Type string `json:"type"` Value string `json:"value"` @@ -782,7 +878,31 @@ func (c *CosmosChain) Start(testName string, ctx context.Context, additionalGene return err } + // Start any sidecar processes that should be running before the chain starts eg, egCtx := errgroup.WithContext(ctx) + for _, s := range c.Sidecars { + s := s + + err = s.containerLifecycle.Running(ctx) + if s.preStart && err != nil { + eg.Go(func() error { + if err := s.CreateContainer(egCtx); err != nil { + return err + } + + if err := s.StartContainer(egCtx); err != nil { + return err + } + + return nil + }) + } + } + if err := eg.Wait(); err != nil { + return err + } + + eg, egCtx = errgroup.WithContext(ctx) for _, n := range chainNodes { n := n eg.Go(func() error { @@ -907,6 +1027,21 @@ func (c *CosmosChain) StopAllNodes(ctx context.Context) error { return eg.Wait() } +// StopAllSidecars stops and removes all long-running containers for sidecar processes. +func (c *CosmosChain) StopAllSidecars(ctx context.Context) error { + var eg errgroup.Group + for _, s := range c.Sidecars { + s := s + eg.Go(func() error { + if err := s.StopContainer(ctx); err != nil { + return err + } + return s.RemoveContainer(ctx) + }) + } + return eg.Wait() +} + // StartAllNodes creates and starts new containers for each node. // Should only be used if the chain has previously been started with .Start. func (c *CosmosChain) StartAllNodes(ctx context.Context) error { @@ -926,6 +1061,60 @@ func (c *CosmosChain) StartAllNodes(ctx context.Context) error { return eg.Wait() } +// StartAllSidecars creates and starts new containers for each sidecar process. +// Should only be used if the chain has previously been started with .Start. +func (c *CosmosChain) StartAllSidecars(ctx context.Context) error { + // prevent client calls during this time + c.findTxMu.Lock() + defer c.findTxMu.Unlock() + var eg errgroup.Group + for _, s := range c.Sidecars { + s := s + + err := s.containerLifecycle.Running(ctx) + if err == nil { + continue + } + + eg.Go(func() error { + if err := s.CreateContainer(ctx); err != nil { + return err + } + return s.StartContainer(ctx) + }) + } + return eg.Wait() +} + +// StartAllValSidecars creates and starts new containers for each validator sidecar process. +// Should only be used if the chain has previously been started with .Start. +func (c *CosmosChain) StartAllValSidecars(ctx context.Context) error { + // prevent client calls during this time + c.findTxMu.Lock() + defer c.findTxMu.Unlock() + var eg errgroup.Group + + for _, v := range c.Validators { + for _, s := range v.Sidecars { + s := s + + err := s.containerLifecycle.Running(ctx) + if err == nil { + continue + } + + eg.Go(func() error { + if err := s.CreateContainer(ctx); err != nil { + return err + } + return s.StartContainer(ctx) + }) + } + } + + return eg.Wait() +} + func (c *CosmosChain) VoteOnProposalAllValidators(ctx context.Context, proposalID string, vote string) error { var eg errgroup.Group for _, n := range c.Nodes() { diff --git a/chain/cosmos/sidecar.go b/chain/cosmos/sidecar.go new file mode 100644 index 000000000..fc12ad64b --- /dev/null +++ b/chain/cosmos/sidecar.go @@ -0,0 +1,178 @@ +package cosmos + +import ( + "context" + "fmt" + "os" + + dockerclient "github.com/docker/docker/client" + "github.com/docker/go-connections/nat" + "github.com/strangelove-ventures/interchaintest/v6/ibc" + "github.com/strangelove-ventures/interchaintest/v6/internal/dockerutil" + "go.uber.org/zap" +) + +type SidecarProcesses []*SidecarProcess + +// SidecarProcess represents a companion process that may be required on a per chain or per validator basis. +type SidecarProcess struct { + log *zap.Logger + + Index int + Chain ibc.Chain + + // If true this process is scoped to a specific validator, otherwise it is scoped at the chain level. + validatorProcess bool + + // If true this process should be started before the chain or validator, otherwise it should be explicitly started after. + preStart bool + + ProcessName string + TestName string + + VolumeName string + DockerClient *dockerclient.Client + NetworkID string + Image ibc.DockerImage + ports nat.PortSet + startCmd []string + homeDir string + + containerLifecycle *dockerutil.ContainerLifecycle +} + +// NewSidecar instantiates a new SidecarProcess. +func NewSidecar( + log *zap.Logger, + validatorProcess bool, + preStart bool, + chain ibc.Chain, + dockerClient *dockerclient.Client, + networkID, processName, testName string, + image ibc.DockerImage, + homeDir string, + index int, + ports []string, + startCmd []string, +) *SidecarProcess { + processPorts := nat.PortSet{} + + for _, port := range ports { + processPorts[nat.Port(port)] = struct{}{} + } + + if homeDir == "" { + homeDir = "/home/sidecar" + } + + s := &SidecarProcess{ + log: log, + Index: index, + Chain: chain, + preStart: preStart, + validatorProcess: validatorProcess, + ProcessName: processName, + TestName: testName, + DockerClient: dockerClient, + NetworkID: networkID, + Image: image, + homeDir: homeDir, + ports: processPorts, + startCmd: startCmd, + } + s.containerLifecycle = dockerutil.NewContainerLifecycle(log, dockerClient, s.Name()) + + return s +} + +// Name returns a string identifier based on if this process is configured to run on a chain level or +// on a per validator level. +func (s *SidecarProcess) Name() string { + if s.validatorProcess { + return fmt.Sprintf("%s-%s-val-%d-%s", s.Chain.Config().ChainID, s.ProcessName, s.Index, dockerutil.SanitizeContainerName(s.TestName)) + } + + return fmt.Sprintf("%s-%s-%d-%s", s.Chain.Config().ChainID, s.ProcessName, s.Index, dockerutil.SanitizeContainerName(s.TestName)) +} + +func (s *SidecarProcess) logger() *zap.Logger { + return s.log.With( + zap.String("process_name", s.ProcessName), + zap.String("test", s.TestName), + ) +} + +func (s *SidecarProcess) CreateContainer(ctx context.Context) error { + return s.containerLifecycle.CreateContainer(ctx, s.TestName, s.NetworkID, s.Image, s.ports, s.Bind(), s.HostName(), s.startCmd) +} + +func (s *SidecarProcess) StartContainer(ctx context.Context) error { + return s.containerLifecycle.StartContainer(ctx) +} + +func (s *SidecarProcess) StopContainer(ctx context.Context) error { + return s.containerLifecycle.StopContainer(ctx) +} + +func (s *SidecarProcess) RemoveContainer(ctx context.Context) error { + return s.containerLifecycle.RemoveContainer(ctx) +} + +// Bind returns the home folder bind point for running the process. +func (s *SidecarProcess) Bind() []string { + return []string{fmt.Sprintf("%s:%s", s.VolumeName, s.HomeDir())} +} + +// HomeDir returns the path name where any configuration files will be written to the Docker filesystem. +func (s *SidecarProcess) HomeDir() string { + return s.homeDir +} + +func (s *SidecarProcess) HostName() string { + return dockerutil.CondenseHostName(s.Name()) +} + +func (s *SidecarProcess) GetHostPorts(ctx context.Context, portIDs ...string) ([]string, error) { + return s.containerLifecycle.GetHostPorts(ctx, portIDs...) +} + +// WriteFile accepts file contents in a byte slice and writes the contents to +// the docker filesystem. relPath describes the location of the file in the +// docker volume relative to the home directory +func (s *SidecarProcess) WriteFile(ctx context.Context, content []byte, relPath string) error { + fw := dockerutil.NewFileWriter(s.logger(), s.DockerClient, s.TestName) + return fw.WriteFile(ctx, s.VolumeName, relPath, content) +} + +// CopyFile adds a file from the host filesystem to the docker filesystem +// relPath describes the location of the file in the docker volume relative to +// the home directory +func (s *SidecarProcess) CopyFile(ctx context.Context, srcPath, dstPath string) error { + content, err := os.ReadFile(srcPath) + if err != nil { + return err + } + return s.WriteFile(ctx, content, dstPath) +} + +// ReadFile reads the contents of a single file at the specified path in the docker filesystem. +// relPath describes the location of the file in the docker volume relative to the home directory. +func (s *SidecarProcess) ReadFile(ctx context.Context, relPath string) ([]byte, error) { + fr := dockerutil.NewFileRetriever(s.logger(), s.DockerClient, s.TestName) + gen, err := fr.SingleFileContent(ctx, s.VolumeName, relPath) + if err != nil { + return nil, fmt.Errorf("failed to read file at %s: %w", relPath, err) + } + return gen, nil +} + +// Exec enables the execution of arbitrary CLI cmds against the process. +func (s *SidecarProcess) Exec(ctx context.Context, cmd []string, env []string) ([]byte, []byte, error) { + job := dockerutil.NewImage(s.logger(), s.DockerClient, s.NetworkID, s.TestName, s.Image.Repository, s.Image.Version) + opts := dockerutil.ContainerOptions{ + Env: env, + Binds: s.Bind(), + } + res := job.Run(ctx, cmd, opts) + return res.Stdout, res.Stderr, res.Err +} diff --git a/ibc/types.go b/ibc/types.go index de3821cce..ee44eded7 100644 --- a/ibc/types.go +++ b/ibc/types.go @@ -51,13 +51,21 @@ type ChainConfig struct { UsingNewGenesisCommand bool `yaml:"using-new-genesis-command"` // Required when the chain requires the chain-id field to be populated for certain commands UsingChainIDFlagCLI bool `yaml:"using-chain-id-flag-cli"` + // Configuration describing additional sidecar processes. + SidecarConfigs []SidecarConfig } func (c ChainConfig) Clone() ChainConfig { x := c + images := make([]DockerImage, len(c.Images)) copy(images, c.Images) x.Images = images + + sidecars := make([]SidecarConfig, len(c.SidecarConfigs)) + copy(sidecars, c.SidecarConfigs) + x.SidecarConfigs = sidecars + return x } @@ -151,6 +159,10 @@ func (c ChainConfig) MergeChainSpecConfig(other ChainConfig) ChainConfig { c.EncodingConfig = other.EncodingConfig } + if len(other.SidecarConfigs) > 0 { + c.SidecarConfigs = append([]SidecarConfig(nil), other.SidecarConfigs...) + } + return c } @@ -169,6 +181,17 @@ func (c ChainConfig) IsFullyConfigured() bool { c.TrustingPeriod != "" } +// SidecarConfig describes the configuration options for instantiating a new sidecar process. +type SidecarConfig struct { + ProcessName string + Image DockerImage + HomeDir string + Ports []string + StartCmd []string + PreStart bool + ValidatorProcess bool +} + type DockerImage struct { Repository string `yaml:"repository"` Version string `yaml:"version"` diff --git a/internal/dockerutil/container_lifecycle.go b/internal/dockerutil/container_lifecycle.go index f3d6a8604..f4c9e007f 100644 --- a/internal/dockerutil/container_lifecycle.go +++ b/internal/dockerutil/container_lifecycle.go @@ -147,3 +147,16 @@ func (c *ContainerLifecycle) GetHostPorts(ctx context.Context, portIDs ...string } return ports, nil } + +// Running will inspect the container and check its state to determine if it is currently running. +// If the container is running nil will be returned, otherwise an error is returned. +func (c *ContainerLifecycle) Running(ctx context.Context) error { + cjson, err := c.client.ContainerInspect(ctx, c.id) + if err != nil { + return err + } + if cjson.State.Running { + return nil + } + return fmt.Errorf("container with name %s and id %s is not running", c.containerName, c.id) +}