Skip to content

Commit

Permalink
feat: add the ability to manage sidecar processes (#465)
Browse files Browse the repository at this point in the history 
* feat: add the ability to manage sidecar processes on a per chain or per validator level

* feat: add the ability to configure sidecars through chain config

* feat: add the ability to check if a container is running

* handle changes in update to docker dep

* add logic for merge and clone of sidecars

* Sidecar home dir and host ports (#665)

* Make it work with horcrux

* Add GetHostPorts

---------

Co-authored-by: Andrew Gouin <[email protected]>
(cherry picked from commit 2d8518c)

# Conflicts:
#	ibc/types.go
  • Loading branch information
@jtieri @mergify
jtieri authored and mergify[bot] committed Aug 17, 2023
1 parent 68266b4 commit d5b207e
Show file tree
Hide file tree
Showing 5 changed files with 476 additions and 0 deletions.
70 changes: 70 additions & 0 deletions chain/cosmos/chain_node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
authTx "github.com/cosmos/cosmos-sdk/x/auth/tx"
paramsutils "github.com/cosmos/cosmos-sdk/x/params/client/utils"
stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
volumetypes "github.com/docker/docker/api/types/volume"
dockerclient "github.com/docker/docker/client"
"github.com/docker/go-connections/nat"
"github.com/strangelove-ventures/interchaintest/v6/ibc"
Expand Down Expand Up @@ -51,6 +52,9 @@ type ChainNode struct {
TestName string
Image ibc.DockerImage

// Additional processes that need to be run on a per-validator basis.
Sidecars SidecarProcesses

lock sync.Mutex
log *zap.Logger

Expand Down Expand Up @@ -121,6 +125,48 @@ func (tn *ChainNode) NewClient(addr string) error {
return nil
}

func (tn *ChainNode) NewSidecarProcess(
ctx context.Context,
preStart bool,
processName string,
cli *dockerclient.Client,
networkID string,
image ibc.DockerImage,
homeDir string,
ports []string,
startCmd []string,
) error {
s := NewSidecar(tn.log, true, preStart, tn.Chain, cli, networkID, processName, tn.TestName, image, homeDir, tn.Index, ports, startCmd)

v, err := cli.VolumeCreate(ctx, volumetypes.CreateOptions{
Labels: map[string]string{
dockerutil.CleanupLabel: tn.TestName,
dockerutil.NodeOwnerLabel: s.Name(),
},
})
if err != nil {
return fmt.Errorf("creating volume for sidecar process: %w", err)
}
s.VolumeName = v.Name

if err := dockerutil.SetVolumeOwner(ctx, dockerutil.VolumeOwnerOptions{
Log: tn.log,

Client: cli,

VolumeName: v.Name,
ImageRef: image.Ref(),
TestName: tn.TestName,
UidGid: image.UidGid,
}); err != nil {
return fmt.Errorf("set volume owner: %w", err)
}

tn.Sidecars = append(tn.Sidecars, s)

return nil
}

// CliContext creates a new Cosmos SDK client context
func (tn *ChainNode) CliContext() client.Context {
cfg := tn.Chain.Config()
Expand Down Expand Up @@ -911,6 +957,20 @@ func (tn *ChainNode) CreateNodeContainer(ctx context.Context) error {
}

func (tn *ChainNode) StartContainer(ctx context.Context) error {
for _, s := range tn.Sidecars {
err := s.containerLifecycle.Running(ctx)

if s.preStart && err != nil {
if err := s.CreateContainer(ctx); err != nil {
return err
}

if err := s.StartContainer(ctx); err != nil {
return err
}
}
}

if err := tn.containerLifecycle.StartContainer(ctx); err != nil {
return err
}
Expand Down Expand Up @@ -943,10 +1003,20 @@ func (tn *ChainNode) StartContainer(ctx context.Context) error {
}

func (tn *ChainNode) StopContainer(ctx context.Context) error {
for _, s := range tn.Sidecars {
if err := s.StopContainer(ctx); err != nil {
return err
}
}
return tn.containerLifecycle.StopContainer(ctx)
}

func (tn *ChainNode) RemoveContainer(ctx context.Context) error {
for _, s := range tn.Sidecars {
if err := s.RemoveContainer(ctx); err != nil {
return err
}
}
return tn.containerLifecycle.RemoveContainer(ctx)
}

Expand Down
189 changes: 189 additions & 0 deletions chain/cosmos/cosmos_chain.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ type CosmosChain struct {
Validators ChainNodes
FullNodes ChainNodes

// Additional processes that need to be run on a per-chain basis.
Sidecars SidecarProcesses

log *zap.Logger
keyring keyring.Keyring
findTxMu sync.Mutex
Expand Down Expand Up @@ -165,6 +168,9 @@ func (c *CosmosChain) Config() ibc.ChainConfig {

// Implements Chain interface
func (c *CosmosChain) Initialize(ctx context.Context, testName string, cli *client.Client, networkID string) error {
if err := c.initializeSidecars(ctx, testName, cli, networkID); err != nil {
return err
}
return c.initializeChainNodes(ctx, testName, cli, networkID)
}

Expand Down Expand Up @@ -543,9 +549,68 @@ func (c *CosmosChain) NewChainNode(
}); err != nil {
return nil, fmt.Errorf("set volume owner: %w", err)
}

for _, cfg := range c.cfg.SidecarConfigs {
if !cfg.ValidatorProcess {
continue
}

err = tn.NewSidecarProcess(ctx, cfg.PreStart, cfg.ProcessName, cli, networkID, cfg.Image, cfg.HomeDir, cfg.Ports, cfg.StartCmd)
if err != nil {
return nil, err
}
}

return tn, nil
}

// NewSidecarProcess constructs a new sidecar process with a docker volume.
func (c *CosmosChain) NewSidecarProcess(
ctx context.Context,
preStart bool,
processName string,
testName string,
cli *client.Client,
networkID string,
image ibc.DockerImage,
homeDir string,
index int,
ports []string,
startCmd []string,
) error {
// Construct the SidecarProcess first so we can access its name.
// The SidecarProcess's VolumeName cannot be set until after we create the volume.
s := NewSidecar(c.log, false, preStart, c, cli, networkID, processName, testName, image, homeDir, index, ports, startCmd)

v, err := cli.VolumeCreate(ctx, volumetypes.CreateOptions{
Labels: map[string]string{
dockerutil.CleanupLabel: testName,
dockerutil.NodeOwnerLabel: s.Name(),
},
})
if err != nil {
return fmt.Errorf("creating volume for sidecar process: %w", err)
}
s.VolumeName = v.Name

if err := dockerutil.SetVolumeOwner(ctx, dockerutil.VolumeOwnerOptions{
Log: c.log,

Client: cli,

VolumeName: v.Name,
ImageRef: image.Ref(),
TestName: testName,
UidGid: image.UidGid,
}); err != nil {
return fmt.Errorf("set volume owner: %w", err)
}

c.Sidecars = append(c.Sidecars, s)

return nil
}

// creates the test node objects required for bootstrapping tests
func (c *CosmosChain) initializeChainNodes(
ctx context.Context,
Expand Down Expand Up @@ -595,6 +660,37 @@ func (c *CosmosChain) initializeChainNodes(
return nil
}

// initializeSidecars creates the sidecar processes that exist at the chain level.
func (c *CosmosChain) initializeSidecars(
ctx context.Context,
testName string,
cli *client.Client,
networkID string,
) error {
eg, egCtx := errgroup.WithContext(ctx)
for i, cfg := range c.cfg.SidecarConfigs {
i := i
cfg := cfg

if cfg.ValidatorProcess {
continue
}

eg.Go(func() error {
err := c.NewSidecarProcess(egCtx, cfg.PreStart, cfg.ProcessName, testName, cli, networkID, cfg.Image, cfg.HomeDir, i, cfg.Ports, cfg.StartCmd)
if err != nil {
return err
}
return nil
})

}
if err := eg.Wait(); err != nil {
return err
}
return nil
}

type GenesisValidatorPubKey struct {
Type string `json:"type"`
Value string `json:"value"`
Expand Down Expand Up @@ -782,7 +878,31 @@ func (c *CosmosChain) Start(testName string, ctx context.Context, additionalGene
return err
}

// Start any sidecar processes that should be running before the chain starts
eg, egCtx := errgroup.WithContext(ctx)
for _, s := range c.Sidecars {
s := s

err = s.containerLifecycle.Running(ctx)
if s.preStart && err != nil {
eg.Go(func() error {
if err := s.CreateContainer(egCtx); err != nil {
return err
}

if err := s.StartContainer(egCtx); err != nil {
return err
}

return nil
})
}
}
if err := eg.Wait(); err != nil {
return err
}

eg, egCtx = errgroup.WithContext(ctx)
for _, n := range chainNodes {
n := n
eg.Go(func() error {
Expand Down Expand Up @@ -907,6 +1027,21 @@ func (c *CosmosChain) StopAllNodes(ctx context.Context) error {
return eg.Wait()
}

// StopAllSidecars stops and removes all long-running containers for sidecar processes.
func (c *CosmosChain) StopAllSidecars(ctx context.Context) error {
var eg errgroup.Group
for _, s := range c.Sidecars {
s := s
eg.Go(func() error {
if err := s.StopContainer(ctx); err != nil {
return err
}
return s.RemoveContainer(ctx)
})
}
return eg.Wait()
}

// StartAllNodes creates and starts new containers for each node.
// Should only be used if the chain has previously been started with .Start.
func (c *CosmosChain) StartAllNodes(ctx context.Context) error {
Expand All @@ -926,6 +1061,60 @@ func (c *CosmosChain) StartAllNodes(ctx context.Context) error {
return eg.Wait()
}

// StartAllSidecars creates and starts new containers for each sidecar process.
// Should only be used if the chain has previously been started with .Start.
func (c *CosmosChain) StartAllSidecars(ctx context.Context) error {
// prevent client calls during this time
c.findTxMu.Lock()
defer c.findTxMu.Unlock()
var eg errgroup.Group
for _, s := range c.Sidecars {
s := s

err := s.containerLifecycle.Running(ctx)
if err == nil {
continue
}

eg.Go(func() error {
if err := s.CreateContainer(ctx); err != nil {
return err
}
return s.StartContainer(ctx)
})
}
return eg.Wait()
}

// StartAllValSidecars creates and starts new containers for each validator sidecar process.
// Should only be used if the chain has previously been started with .Start.
func (c *CosmosChain) StartAllValSidecars(ctx context.Context) error {
// prevent client calls during this time
c.findTxMu.Lock()
defer c.findTxMu.Unlock()
var eg errgroup.Group

for _, v := range c.Validators {
for _, s := range v.Sidecars {
s := s

err := s.containerLifecycle.Running(ctx)
if err == nil {
continue
}

eg.Go(func() error {
if err := s.CreateContainer(ctx); err != nil {
return err
}
return s.StartContainer(ctx)
})
}
}

return eg.Wait()
}

func (c *CosmosChain) VoteOnProposalAllValidators(ctx context.Context, proposalID string, vote string) error {
var eg errgroup.Group
for _, n := range c.Nodes() {
Expand Down
Loading

0 comments on commit d5b207e

Please sign in to comment.