Skip to content

Commit

Permalink
fix: improve warnings, cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
bohdand-weka committed Feb 1, 2024
1 parent 0e9b7fa commit 7f0571e
Show file tree
Hide file tree
Showing 4 changed files with 130 additions and 108 deletions.
45 changes: 0 additions & 45 deletions internal/local/chart/events.go

This file was deleted.

57 changes: 24 additions & 33 deletions internal/local/chart/install.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,43 +6,15 @@ import (
"fmt"
"strings"

helmclient "github.com/mittwald/go-helm-client"
"k8s.io/apimachinery/pkg/util/wait"

"github.com/weka/gohomecli/internal/utils"
)

var ErrTimeout = errors.New("timeout")

func Install(ctx context.Context, opts *HelmOptions) error {
namespace := ReleaseNamespace
if opts.NamespaceOverride != "" {
namespace = opts.NamespaceOverride
}

go watchEvents(ctx, opts)

logger.Info().
Str("namespace", namespace).
Str("kubeContext", opts.KubeContext).
Msg("Configuring helm client")

// kubeContext override isn't working - https://github.com/mittwald/go-helm-client/issues/127
client, err := helmclient.NewClientFromKubeConf(&helmclient.KubeConfClientOptions{
Options: &helmclient.Options{
Namespace: namespace,
DebugLog: func(format string, v ...interface{}) {
logger.Debug().Msgf(format, v...)
},
Output: utils.NewWritterFunc(func(b []byte) {
logger.Info().Msg(string(b))
}),
},
KubeContext: opts.KubeContext,
KubeConfig: opts.KubeConfig,
})
client, err := NewHelmClient(ctx, opts)
if err != nil {
return fmt.Errorf("failed configuring helm client: %w", err)
return fmt.Errorf("helm client: %w", err)
}

spec, err := chartSpec(client, opts)
Expand All @@ -56,13 +28,32 @@ func Install(ctx context.Context, opts *HelmOptions) error {
Str("release", spec.ReleaseName).
Msg("Installing chart")

warnings, cancel, err := watchWarningEvents(ctx, spec.Namespace, opts.KubeConfig)
if err != nil {
logger.Warn().Err(err).Msg("Failed to watch events")
}

release, err := client.InstallChart(ctx, spec, nil)

// stop watching for events
if cancel != nil {
cancel()
}

if err != nil {
// if it's not canceled, print warnings
if !errors.Is(err, context.Canceled) && len(warnings) > 0 {
logger.Info().Msg("Received next warnings:")
for warn := range warnings {
logger.Warn().Str("name", warn.Name).Msg(warn.Message)
}
}

if isTimeoutErr(err) {
return fmt.Errorf("failed installing chart: %w", ErrTimeout)
return errors.Join(ErrTimeout, err)
}

return fmt.Errorf("failed installing chart: %w", err)
return err
}

logger.Info().Msg(release.Info.Notes)
Expand All @@ -80,7 +71,7 @@ func isTimeoutErr(err error) bool {
return false
}
return true
case strings.Contains(err.Error(), "would exceed context deadline"):
case strings.Contains(err.Error(), "context deadline"):
// there is no dedicated error in kubernetes rate limiter
// so we need to check the error message
return true
Expand Down
80 changes: 80 additions & 0 deletions internal/local/chart/kube.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,17 @@
package chart

import (
"context"
"fmt"
"os"
"path/filepath"

helmclient "github.com/mittwald/go-helm-client"
"github.com/weka/gohomecli/internal/utils"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
)

const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml"
Expand Down Expand Up @@ -31,3 +39,75 @@ func ReadKubeConfig(kubeConfigPath string) ([]byte, error) {

return kubeConfig, nil
}

func NewHelmClient(ctx context.Context, opts *HelmOptions) (helmclient.Client, error) {
namespace := ReleaseNamespace
if opts.NamespaceOverride != "" {
namespace = opts.NamespaceOverride
}

logger.Info().
Str("namespace", namespace).
Str("kubeContext", opts.KubeContext).
Msg("Configuring helm client")

// kubeContext override isn't working - https://github.com/mittwald/go-helm-client/issues/127
return helmclient.NewClientFromKubeConf(&helmclient.KubeConfClientOptions{
Options: &helmclient.Options{
Namespace: namespace,
DebugLog: func(format string, v ...interface{}) {
logger.Debug().Msgf(format, v...)
},
Output: utils.NewWritterFunc(func(b []byte) {
logger.Info().Msg(string(b))
}),
},
KubeContext: opts.KubeContext,
KubeConfig: opts.KubeConfig,
})
}

type warningEvent struct {
Name string
Message string
}

// watchWarningEvents watches for warning events
func watchWarningEvents(ctx context.Context, namespace string, kubeconfig []byte) (chan warningEvent, func(), error) {
cfg, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig)
if err != nil {
return nil, nil, err
}

k8s, err := kubernetes.NewForConfig(cfg)
if err != nil {
return nil, nil, err
}

watcher, err := k8s.CoreV1().Events(namespace).
Watch(ctx, v1.ListOptions{TypeMeta: v1.TypeMeta{Kind: "Pod"}})
if err != nil {
return nil, nil, err
}

ch := make(chan warningEvent, 10000)

go func() {
for evt := range watcher.ResultChan() {
switch ev := evt.Object.(type) {
case *corev1.Event:
if ev.Type == "Warning" && ev.Reason != "BackOff" {
logger.Debug().Str("name", ev.Name).Msg(ev.Message)
ch <- warningEvent{Name: ev.Name, Message: ev.Message}
}
case *v1.Status:
logger.Debug().Msg(ev.Message)
default:
logger.Debug().Msgf("Uknown event type: %T", ev)
}
}
close(ch)
}()

return ch, watcher.Stop, nil
}
56 changes: 26 additions & 30 deletions internal/local/chart/upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,41 +2,14 @@ package chart

import (
"context"
"errors"
"fmt"

helmclient "github.com/mittwald/go-helm-client"
"github.com/weka/gohomecli/internal/utils"
)

func Upgrade(ctx context.Context, opts *HelmOptions, debug bool) error {
namespace := ReleaseNamespace
if opts.NamespaceOverride != "" {
namespace = opts.NamespaceOverride
}

go watchEvents(ctx, opts)

logger.Info().
Str("namespace", namespace).
Str("kubeContext", opts.KubeContext).
Msg("Configuring helm client")

// kubeContext override isn't working - https://github.com/mittwald/go-helm-client/issues/127
client, err := helmclient.NewClientFromKubeConf(&helmclient.KubeConfClientOptions{
Options: &helmclient.Options{
Namespace: namespace,
DebugLog: func(format string, v ...interface{}) {
logger.Debug().Msgf(format, v...)
},
Output: utils.NewWritterFunc(func(b []byte) {
logger.Info().Msg(string(b))
}),
},
KubeContext: opts.KubeContext,
KubeConfig: opts.KubeConfig,
})
client, err := NewHelmClient(ctx, opts)
if err != nil {
return fmt.Errorf("failed configuring helm client: %w", err)
return fmt.Errorf("helm client: %w", err)
}

spec, err := chartSpec(client, opts)
Expand All @@ -50,14 +23,37 @@ func Upgrade(ctx context.Context, opts *HelmOptions, debug bool) error {
Str("release", spec.ReleaseName).
Msg("Upgrading chart")

warnings, cancel, err := watchWarningEvents(ctx, spec.Namespace, opts.KubeConfig)
if err != nil {
logger.Warn().Err(err).Msg("Failed to watch events")
}

release, err := client.UpgradeChart(ctx, spec, nil)

if cancel != nil {
cancel()
}

if err != nil {
// if it's not canceled, print warnings
if !errors.Is(err, context.Canceled) && len(warnings) > 0 {
logger.Info().Msg("Received next warnings:")
for warn := range warnings {
logger.Warn().Str("name", warn.Name).Msg(warn.Message)
}
}

if !debug {
logger.Warn().Msg("Rolling back release")
if err := client.RollbackRelease(spec); err != nil {
logger.Error().Err(err).Msg("Rollback failed")
}
}

if isTimeoutErr(err) {
return errors.Join(ErrTimeout, err)
}

logger.Error().Err(err).Msg("Upgrade failed")
return err
}
Expand Down

0 comments on commit 7f0571e

Please sign in to comment.