feat: add the ability to manage sidecar processes (backport #465)

chain/cosmos/chain_node.go

@@ -23,6 +23,7 @@ import (
 	authTx "github.com/cosmos/cosmos-sdk/x/auth/tx"
 	paramsutils "github.com/cosmos/cosmos-sdk/x/params/client/utils"
 	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
+	volumetypes "github.com/docker/docker/api/types/volume"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/docker/go-connections/nat"
 	"github.com/strangelove-ventures/interchaintest/v6/ibc"
@@ -51,6 +52,9 @@ type ChainNode struct {
 	TestName     string
 	Image        ibc.DockerImage
 
+	// Additional processes that need to be run on a per-validator basis.
+	Sidecars SidecarProcesses
+
 	lock sync.Mutex
 	log  *zap.Logger
 
@@ -121,6 +125,48 @@ func (tn *ChainNode) NewClient(addr string) error {
 	return nil
 }
 
+func (tn *ChainNode) NewSidecarProcess(
+	ctx context.Context,
+	preStart bool,
+	processName string,
+	cli *dockerclient.Client,
+	networkID string,
+	image ibc.DockerImage,
+	homeDir string,
+	ports []string,
+	startCmd []string,
+) error {
+	s := NewSidecar(tn.log, true, preStart, tn.Chain, cli, networkID, processName, tn.TestName, image, homeDir, tn.Index, ports, startCmd)
+
+	v, err := cli.VolumeCreate(ctx, volumetypes.CreateOptions{
+		Labels: map[string]string{
+			dockerutil.CleanupLabel:   tn.TestName,
+			dockerutil.NodeOwnerLabel: s.Name(),
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("creating volume for sidecar process: %w", err)
+	}
+	s.VolumeName = v.Name
+
+	if err := dockerutil.SetVolumeOwner(ctx, dockerutil.VolumeOwnerOptions{
+		Log: tn.log,
+
+		Client: cli,
+
+		VolumeName: v.Name,
+		ImageRef:   image.Ref(),
+		TestName:   tn.TestName,
+		UidGid:     image.UidGid,
+	}); err != nil {
+		return fmt.Errorf("set volume owner: %w", err)
+	}
+
+	tn.Sidecars = append(tn.Sidecars, s)
+
+	return nil
+}
+
 // CliContext creates a new Cosmos SDK client context
 func (tn *ChainNode) CliContext() client.Context {
 	cfg := tn.Chain.Config()
@@ -911,6 +957,20 @@ func (tn *ChainNode) CreateNodeContainer(ctx context.Context) error {
 }
 
 func (tn *ChainNode) StartContainer(ctx context.Context) error {
+	for _, s := range tn.Sidecars {
+		err := s.containerLifecycle.Running(ctx)
+
+		if s.preStart && err != nil {
+			if err := s.CreateContainer(ctx); err != nil {
+				return err
+			}
+
+			if err := s.StartContainer(ctx); err != nil {
+				return err
+			}
+		}
+	}
+
 	if err := tn.containerLifecycle.StartContainer(ctx); err != nil {
 		return err
 	}
@@ -943,10 +1003,20 @@ func (tn *ChainNode) StartContainer(ctx context.Context) error {
 }
 
 func (tn *ChainNode) StopContainer(ctx context.Context) error {
+	for _, s := range tn.Sidecars {
+		if err := s.StopContainer(ctx); err != nil {
+			return err
+		}
+	}
 	return tn.containerLifecycle.StopContainer(ctx)
 }
 
 func (tn *ChainNode) RemoveContainer(ctx context.Context) error {
+	for _, s := range tn.Sidecars {
+		if err := s.RemoveContainer(ctx); err != nil {
+			return err
+		}
+	}
 	return tn.containerLifecycle.RemoveContainer(ctx)
 }
 

chain/cosmos/cosmos_chain.go

@@ -43,6 +43,9 @@ type CosmosChain struct {
 	Validators    ChainNodes
 	FullNodes     ChainNodes
 
+	// Additional processes that need to be run on a per-chain basis.
+	Sidecars SidecarProcesses
+
 	log      *zap.Logger
 	keyring  keyring.Keyring
 	findTxMu sync.Mutex
@@ -165,6 +168,9 @@ func (c *CosmosChain) Config() ibc.ChainConfig {
 
 // Implements Chain interface
 func (c *CosmosChain) Initialize(ctx context.Context, testName string, cli *client.Client, networkID string) error {
+	if err := c.initializeSidecars(ctx, testName, cli, networkID); err != nil {
+		return err
+	}
 	return c.initializeChainNodes(ctx, testName, cli, networkID)
 }
 
@@ -543,9 +549,68 @@ func (c *CosmosChain) NewChainNode(
 	}); err != nil {
 		return nil, fmt.Errorf("set volume owner: %w", err)
 	}
+
+	for _, cfg := range c.cfg.SidecarConfigs {
+		if !cfg.ValidatorProcess {
+			continue
+		}
+
+		err = tn.NewSidecarProcess(ctx, cfg.PreStart, cfg.ProcessName, cli, networkID, cfg.Image, cfg.HomeDir, cfg.Ports, cfg.StartCmd)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return tn, nil
 }
 
+// NewSidecarProcess constructs a new sidecar process with a docker volume.
+func (c *CosmosChain) NewSidecarProcess(
+	ctx context.Context,
+	preStart bool,
+	processName string,
+	testName string,
+	cli *client.Client,
+	networkID string,
+	image ibc.DockerImage,
+	homeDir string,
+	index int,
+	ports []string,
+	startCmd []string,
+) error {
+	// Construct the SidecarProcess first so we can access its name.
+	// The SidecarProcess's VolumeName cannot be set until after we create the volume.
+	s := NewSidecar(c.log, false, preStart, c, cli, networkID, processName, testName, image, homeDir, index, ports, startCmd)
+
+	v, err := cli.VolumeCreate(ctx, volumetypes.CreateOptions{
+		Labels: map[string]string{
+			dockerutil.CleanupLabel:   testName,
+			dockerutil.NodeOwnerLabel: s.Name(),
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("creating volume for sidecar process: %w", err)
+	}
+	s.VolumeName = v.Name
+
+	if err := dockerutil.SetVolumeOwner(ctx, dockerutil.VolumeOwnerOptions{
+		Log: c.log,
+
+		Client: cli,
+
+		VolumeName: v.Name,
+		ImageRef:   image.Ref(),
+		TestName:   testName,
+		UidGid:     image.UidGid,
+	}); err != nil {
+		return fmt.Errorf("set volume owner: %w", err)
+	}
+
+	c.Sidecars = append(c.Sidecars, s)
+
+	return nil
+}
+
 // creates the test node objects required for bootstrapping tests
 func (c *CosmosChain) initializeChainNodes(
 	ctx context.Context,
@@ -595,6 +660,37 @@ func (c *CosmosChain) initializeChainNodes(
 	return nil
 }
 
+// initializeSidecars creates the sidecar processes that exist at the chain level.
+func (c *CosmosChain) initializeSidecars(
+	ctx context.Context,
+	testName string,
+	cli *client.Client,
+	networkID string,
+) error {
+	eg, egCtx := errgroup.WithContext(ctx)
+	for i, cfg := range c.cfg.SidecarConfigs {
+		i := i
+		cfg := cfg
+
+		if cfg.ValidatorProcess {
+			continue
+		}
+
+		eg.Go(func() error {
+			err := c.NewSidecarProcess(egCtx, cfg.PreStart, cfg.ProcessName, testName, cli, networkID, cfg.Image, cfg.HomeDir, i, cfg.Ports, cfg.StartCmd)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+
+	}
+	if err := eg.Wait(); err != nil {
+		return err
+	}
+	return nil
+}
+
 type GenesisValidatorPubKey struct {
 	Type  string `json:"type"`
 	Value string `json:"value"`
@@ -782,7 +878,31 @@ func (c *CosmosChain) Start(testName string, ctx context.Context, additionalGene
 		return err
 	}
 
+	// Start any sidecar processes that should be running before the chain starts
 	eg, egCtx := errgroup.WithContext(ctx)
+	for _, s := range c.Sidecars {
+		s := s
+
+		err = s.containerLifecycle.Running(ctx)
+		if s.preStart && err != nil {
+			eg.Go(func() error {
+				if err := s.CreateContainer(egCtx); err != nil {
+					return err
+				}
+
+				if err := s.StartContainer(egCtx); err != nil {
+					return err
+				}
+
+				return nil
+			})
+		}
+	}
+	if err := eg.Wait(); err != nil {
+		return err
+	}
+
+	eg, egCtx = errgroup.WithContext(ctx)
 	for _, n := range chainNodes {
 		n := n
 		eg.Go(func() error {
@@ -907,6 +1027,21 @@ func (c *CosmosChain) StopAllNodes(ctx context.Context) error {
 	return eg.Wait()
 }
 
+// StopAllSidecars stops and removes all long-running containers for sidecar processes.
+func (c *CosmosChain) StopAllSidecars(ctx context.Context) error {
+	var eg errgroup.Group
+	for _, s := range c.Sidecars {
+		s := s
+		eg.Go(func() error {
+			if err := s.StopContainer(ctx); err != nil {
+				return err
+			}
+			return s.RemoveContainer(ctx)
+		})
+	}
+	return eg.Wait()
+}
+
 // StartAllNodes creates and starts new containers for each node.
 // Should only be used if the chain has previously been started with .Start.
 func (c *CosmosChain) StartAllNodes(ctx context.Context) error {
@@ -926,6 +1061,60 @@ func (c *CosmosChain) StartAllNodes(ctx context.Context) error {
 	return eg.Wait()
 }
 
+// StartAllSidecars creates and starts new containers for each sidecar process.
+// Should only be used if the chain has previously been started with .Start.
+func (c *CosmosChain) StartAllSidecars(ctx context.Context) error {
+	// prevent client calls during this time
+	c.findTxMu.Lock()
+	defer c.findTxMu.Unlock()
+	var eg errgroup.Group
+	for _, s := range c.Sidecars {
+		s := s
+
+		err := s.containerLifecycle.Running(ctx)
+		if err == nil {
+			continue
+		}
+
+		eg.Go(func() error {
+			if err := s.CreateContainer(ctx); err != nil {
+				return err
+			}
+			return s.StartContainer(ctx)
+		})
+	}
+	return eg.Wait()
+}
+
+// StartAllValSidecars creates and starts new containers for each validator sidecar process.
+// Should only be used if the chain has previously been started with .Start.
+func (c *CosmosChain) StartAllValSidecars(ctx context.Context) error {
+	// prevent client calls during this time
+	c.findTxMu.Lock()
+	defer c.findTxMu.Unlock()
+	var eg errgroup.Group
+
+	for _, v := range c.Validators {
+		for _, s := range v.Sidecars {
+			s := s
+
+			err := s.containerLifecycle.Running(ctx)
+			if err == nil {
+				continue
+			}
+
+			eg.Go(func() error {
+				if err := s.CreateContainer(ctx); err != nil {
+					return err
+				}
+				return s.StartContainer(ctx)
+			})
+		}
+	}
+
+	return eg.Wait()
+}
+
 func (c *CosmosChain) VoteOnProposalAllValidators(ctx context.Context, proposalID string, vote string) error {
 	var eg errgroup.Group
 	for _, n := range c.Nodes() {