Skip to content

Commit

Permalink
juniper config mode retry and error handling
Browse files Browse the repository at this point in the history
In rare occasions, getting into config mode can take some time.
Change here adds retry in case of privilege error by Scrapligo.
Scrapligo throws this when config mode is not yet available. Also
adds a bit better error handling and printing pod information when
certificate push failure occurs.

Also gets rid of /sys/class mount which is no longer needed.
  • Loading branch information
nitinsoniism committed Aug 21, 2023
1 parent 65d0d47 commit 3d26874
Showing 1 changed file with 44 additions and 26 deletions.
70 changes: 44 additions & 26 deletions topo/node/juniper/juniper.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ var (
certGenTimeout = 10 * time.Minute
// Time between polls
certGenRetrySleep = 30 * time.Second
// Wait for config mode
configModeTimeout = 10 * time.Minute
// Time between polls - config mode
configModeRetrySleep = 30 * time.Second
)

const (
Expand Down Expand Up @@ -109,6 +113,39 @@ func (n *Node) GRPCConfig() []string {
}
}

// Waits and retries until CLI config mode is up and config is applied
func (n *Node) waitConfigInfraReadyAndPushConfigs(configs []string) error {

Check failure on line 117 in topo/node/juniper/juniper.go

View workflow job for this annotation

GitHub Actions / lint

unnecessary leading newline (whitespace)

log.Infof("Waiting for config to be pushed (timeout: %v) node %s", configModeTimeout, n.Name())
start := time.Now()
for time.Since(start) < configModeTimeout {
multiresp, err := n.cliConn.SendConfigs(configs)
if err != nil {
if strings.Contains(err.Error(), "errPrivilegeError") {
log.Infof("Config mode not ready. Retrying in %v. Node %s, Resp %v", configModeRetrySleep, n.Name(), err)
} else {
return fmt.Errorf("failed pushing configs: %v", err)
}
} else {
for _, resp := range multiresp.Responses {
if resp.Failed != nil {
return resp.Failed
}
if strings.Contains(resp.Result, "commit complete") {
log.Infof("Config mode ready. Config commit done. Node %s", n.Name())
return nil
}
if strings.Contains(resp.Result, "error:") {
log.Infof("Config mode not ready. Retrying in %v. Node %s Response %s", certGenRetrySleep, n.Name(), multiresp.JoinedResult())
}
}
}
time.Sleep(configModeRetrySleep)
}

return fmt.Errorf("failed sending configs")
}

// Waits and retries until Cert infra is up and certs are applied
func (n *Node) waitCertInfraReadyAndPushCert() error {
selfSigned := n.Proto.GetConfig().GetCert().GetSelfSigned()
Expand All @@ -119,7 +156,7 @@ func (n *Node) waitCertInfraReadyAndPushCert() error {
selfSigned.GetCertName()),
}

log.Infof("Waiting for certificates to be pushed (timeout: %v)", certGenTimeout)
log.Infof("Waiting for certificates to be pushed (timeout: %v) node %s", certGenTimeout, n.Name())
start := time.Now()
for time.Since(start) < certGenTimeout {
multiresp, err := n.cliConn.SendCommands(commands)
Expand All @@ -130,13 +167,13 @@ func (n *Node) waitCertInfraReadyAndPushCert() error {
if resp.Failed != nil {
return resp.Failed
}
if strings.Contains(resp.Result, "error:") {
log.Infof("Cert infra isn't ready. Retrying in %v. Response %s", certGenRetrySleep, multiresp.JoinedResult())
}
if strings.Contains(resp.Result, "successfully") {
log.Infof("Cert Infra ready. Configured Certs. Response %s", multiresp.JoinedResult())
log.Infof("Cert Infra ready. Configured Certs. Node %s, Response %s", n.Name(), multiresp.JoinedResult())
return nil
}
if strings.Contains(resp.Result, "error:") {
log.Infof("Cert infra isn't ready. Retrying in %v. Node %s Response %s", certGenRetrySleep, n.Name(), multiresp.JoinedResult())
}
}
time.Sleep(certGenRetrySleep)
}
Expand Down Expand Up @@ -186,13 +223,8 @@ func (n *Node) GenerateSelfSigned(ctx context.Context) error {
}

// Send gRPC config
resp, err := n.cliConn.SendConfigs(n.GRPCConfig())
if err != nil {
return err
}

if resp.Failed != nil {
return resp.Failed
if err := n.waitConfigInfraReadyAndPushConfigs(n.GRPCConfig()); err != nil {
return fmt.Errorf("failed sending grpc config commands - self-signed-cert: %v", err)
}

log.Infof("%s - finished cert generation", n.Name())
Expand Down Expand Up @@ -378,11 +410,6 @@ func (n *Node) Create(ctx context.Context) error {
ReadOnly: false,
MountPath: "/tmp",
},
{
Name: fmt.Sprintf("%s-sys-class-mount", pb.Name),
ReadOnly: false,
MountPath: "/sys/class",
},
{
Name: fmt.Sprintf("%s-dev-shm-mount", pb.Name),
ReadOnly: false,
Expand Down Expand Up @@ -412,15 +439,6 @@ func (n *Node) Create(ctx context.Context) error {
},
},
},
{
Name: fmt.Sprintf("%s-sys-class-mount", pb.Name),
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/sys/class",
Type: &hpd,
},
},
},
{
Name: fmt.Sprintf("%s-dev-shm-mount", pb.Name),
VolumeSource: corev1.VolumeSource{
Expand Down

0 comments on commit 3d26874

Please sign in to comment.