From d4ff1f317c3828e586554b1158ad5e59e78ed509 Mon Sep 17 00:00:00 2001 From: Stefan Benz <46600784+stebenz@users.noreply.github.com> Date: Wed, 4 Mar 2020 12:46:07 +0100 Subject: [PATCH] fix(logs): fixed labels that are used for log collecting and added attribute (#68) * fix(charts): added checking if newer version of the chart are available * fix(logs): fixed labels that are used for log collecting * fix(log): added attribute to log --- .../applications/ambassador/helm/helm.go | 2 +- .../applications/ambassador/logs/logs.go | 7 +- .../applications/argocd/logs/logs.go | 7 +- .../applications/grafana/logs/logs.go | 7 +- .../kubestatemetrics/logs/logs.go | 7 +- .../applications/prometheus/logs/logs.go | 7 +- .../prometheusnodeexporter/logs/logs.go | 7 +- .../prometheusoperator/logs/logs.go | 7 +- internal/templator/helm/chart/fetch/fetch.go | 6 +- internal/templator/helm/chart/fetch/index.go | 88 +++++++++++++++++++ 10 files changed, 122 insertions(+), 23 deletions(-) create mode 100644 internal/templator/helm/chart/fetch/index.go diff --git a/internal/bundle/application/applications/ambassador/helm/helm.go b/internal/bundle/application/applications/ambassador/helm/helm.go index b9f49bb..ce3b2b9 100644 --- a/internal/bundle/application/applications/ambassador/helm/helm.go +++ b/internal/bundle/application/applications/ambassador/helm/helm.go @@ -15,7 +15,7 @@ func GetChartInfo() *chart.Chart { func GetImageTags() map[string]string { return map[string]string{ - "quay.io/datawire/aes": "1.2.0", + "quay.io/datawire/aes": "1.2.1", "prom/statsd-exporter": "v0.8.1", } } diff --git a/internal/bundle/application/applications/ambassador/logs/logs.go b/internal/bundle/application/applications/ambassador/logs/logs.go index 9056870..adb5bb6 100644 --- a/internal/bundle/application/applications/ambassador/logs/logs.go +++ b/internal/bundle/application/applications/ambassador/logs/logs.go @@ -1,13 +1,14 @@ package logs import ( - "github.com/caos/boom/internal/bundle/application/applications/ambassador/info" "github.com/caos/boom/internal/bundle/application/applications/loggingoperator/logging" - "github.com/caos/boom/internal/labels" ) func GetFlow(outputs []string) *logging.FlowConfig { - ls := labels.GetApplicationLabels(info.GetName()) + ls := map[string]string{ + "app.kubernetes.io/instance": "ambassador", + "app.kubernetes.io/name": "ambassador", + } return &logging.FlowConfig{ Name: "flow-ambassador", diff --git a/internal/bundle/application/applications/argocd/logs/logs.go b/internal/bundle/application/applications/argocd/logs/logs.go index aacf406..53638d2 100644 --- a/internal/bundle/application/applications/argocd/logs/logs.go +++ b/internal/bundle/application/applications/argocd/logs/logs.go @@ -1,13 +1,14 @@ package logs import ( - "github.com/caos/boom/internal/bundle/application/applications/argocd/info" "github.com/caos/boom/internal/bundle/application/applications/loggingoperator/logging" - "github.com/caos/boom/internal/labels" ) func GetFlow(outputs []string) *logging.FlowConfig { - ls := labels.GetApplicationLabels(info.GetName()) + ls := map[string]string{ + "app.kubernetes.io/instance": "argocd", + "app.kubernetes.io/part-of": "argocd", + } return &logging.FlowConfig{ Name: "flow-argocd", diff --git a/internal/bundle/application/applications/grafana/logs/logs.go b/internal/bundle/application/applications/grafana/logs/logs.go index ed48a03..8f47d06 100644 --- a/internal/bundle/application/applications/grafana/logs/logs.go +++ b/internal/bundle/application/applications/grafana/logs/logs.go @@ -1,13 +1,14 @@ package logs import ( - "github.com/caos/boom/internal/bundle/application/applications/grafana/info" "github.com/caos/boom/internal/bundle/application/applications/loggingoperator/logging" - "github.com/caos/boom/internal/labels" ) func GetFlow(outputs []string) *logging.FlowConfig { - ls := labels.GetApplicationLabels(info.GetName()) + ls := map[string]string{ + "app.kubernetes.io/instance": "grafana", + "app.kubernetes.io/name": "grafana", + } return &logging.FlowConfig{ Name: "flow-grafana", diff --git a/internal/bundle/application/applications/kubestatemetrics/logs/logs.go b/internal/bundle/application/applications/kubestatemetrics/logs/logs.go index 59e2eeb..cd22c35 100644 --- a/internal/bundle/application/applications/kubestatemetrics/logs/logs.go +++ b/internal/bundle/application/applications/kubestatemetrics/logs/logs.go @@ -1,13 +1,14 @@ package logs import ( - "github.com/caos/boom/internal/bundle/application/applications/kubestatemetrics/info" "github.com/caos/boom/internal/bundle/application/applications/loggingoperator/logging" - "github.com/caos/boom/internal/labels" ) func GetFlow(outputs []string) *logging.FlowConfig { - ls := labels.GetApplicationLabels(info.GetName()) + ls := map[string]string{ + "app.kubernetes.io/instance": "kube-state-metrics", + "app.kubernetes.io/name": "kube-state-metrics", + } return &logging.FlowConfig{ Name: "flow-kube-state-metrics", diff --git a/internal/bundle/application/applications/prometheus/logs/logs.go b/internal/bundle/application/applications/prometheus/logs/logs.go index f7d4921..fb5b99f 100644 --- a/internal/bundle/application/applications/prometheus/logs/logs.go +++ b/internal/bundle/application/applications/prometheus/logs/logs.go @@ -2,12 +2,13 @@ package logs import ( "github.com/caos/boom/internal/bundle/application/applications/loggingoperator/logging" - "github.com/caos/boom/internal/bundle/application/applications/prometheus/info" - "github.com/caos/boom/internal/labels" ) func GetFlow(outputs []string) *logging.FlowConfig { - ls := labels.GetApplicationLabels(info.GetName()) + ls := map[string]string{ + "app": "prometheus", + "prometheus": "caos-prometheus", + } return &logging.FlowConfig{ Name: "flow-prometheus", diff --git a/internal/bundle/application/applications/prometheusnodeexporter/logs/logs.go b/internal/bundle/application/applications/prometheusnodeexporter/logs/logs.go index f9d1421..ecd4512 100644 --- a/internal/bundle/application/applications/prometheusnodeexporter/logs/logs.go +++ b/internal/bundle/application/applications/prometheusnodeexporter/logs/logs.go @@ -2,12 +2,13 @@ package logs import ( "github.com/caos/boom/internal/bundle/application/applications/loggingoperator/logging" - "github.com/caos/boom/internal/bundle/application/applications/prometheusnodeexporter/info" - "github.com/caos/boom/internal/labels" ) func GetFlow(outputs []string) *logging.FlowConfig { - ls := labels.GetApplicationLabels(info.GetName()) + ls := map[string]string{ + "release": "prometheus-node-exporter", + "app": "prometheus-node-exporter", + } return &logging.FlowConfig{ Name: "flow-prometheus-node-exporter", diff --git a/internal/bundle/application/applications/prometheusoperator/logs/logs.go b/internal/bundle/application/applications/prometheusoperator/logs/logs.go index 3066941..82e9698 100644 --- a/internal/bundle/application/applications/prometheusoperator/logs/logs.go +++ b/internal/bundle/application/applications/prometheusoperator/logs/logs.go @@ -2,12 +2,13 @@ package logs import ( "github.com/caos/boom/internal/bundle/application/applications/loggingoperator/logging" - "github.com/caos/boom/internal/bundle/application/applications/prometheusoperator/info" - "github.com/caos/boom/internal/labels" ) func GetFlow(outputs []string) *logging.FlowConfig { - ls := labels.GetApplicationLabels(info.GetName()) + ls := map[string]string{ + "release": "prometheus-operator", + "app": "prometheus-operator-operator", + } return &logging.FlowConfig{ Name: "flow-prometheus-operator", diff --git a/internal/templator/helm/chart/fetch/fetch.go b/internal/templator/helm/chart/fetch/fetch.go index 1e877bb..de790e0 100644 --- a/internal/templator/helm/chart/fetch/fetch.go +++ b/internal/templator/helm/chart/fetch/fetch.go @@ -83,12 +83,16 @@ func All(monitor mntr.Monitor, basePath string) error { return err } + monitor.Info("Checking newer chart versions") + if err := CompareVersions(monitor, basePath, charts); err != nil { + return err + } + monitor.Info("Fetching all charts") for _, chart := range charts { if err := fetch(monitor, basePath, chart); err != nil { return err } - } return nil } diff --git a/internal/templator/helm/chart/fetch/index.go b/internal/templator/helm/chart/fetch/index.go new file mode 100644 index 0000000..698248f --- /dev/null +++ b/internal/templator/helm/chart/fetch/index.go @@ -0,0 +1,88 @@ +package fetch + +import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/caos/boom/internal/helper" + "github.com/caos/orbiter/mntr" + "gopkg.in/yaml.v3" +) + +type index struct { + APIVersion string `yaml:"apiVersion"` + Entries map[string][]entry `yaml:"entries"` +} + +type entry struct { + Version string `yaml:"version"` + AppVersion string `yaml:"appVersion"` +} + +func CompareVersions(monitor mntr.Monitor, basePath string, charts []*ChartInfo) error { + + indexFolderPathAbs, err := helper.GetAbsPath(basePath, "helm", "repository", "cache") + if err != nil { + return err + } + + indexFiles := make(map[string]*index, 0) + err = filepath.Walk(indexFolderPathAbs, func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + return nil + } + + data, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + var indexFile index + if err := yaml.Unmarshal(data, &indexFile); err != nil { + return err + } + + indexFiles[strings.TrimSuffix(info.Name(), "-index.yaml")] = &indexFile + // for k, v := range indexFile.Entries { + // indexFiles[strings.TrimSuffix(info.Name(), "-index.yaml")].Entries[k] = v + // } + return nil + }) + if err != nil { + return err + } + for _, chart := range charts { + indexFile := indexFiles[chart.IndexName] + for _, entry := range indexFile.Entries[chart.Name] { + entryParts := strings.Split(entry.Version, ".") + entryPartsInt := make([]int, 3) + for k, v := range entryParts { + entryPartsInt[k], _ = strconv.Atoi(v) + } + + chartParts := strings.Split(chart.Version, ".") + chartPartsInt := make([]int, 3) + for k, v := range chartParts { + chartPartsInt[k], _ = strconv.Atoi(v) + } + if entryPartsInt[0] > chartPartsInt[0] || + (entryPartsInt[0] == chartPartsInt[0] && entryPartsInt[1] > chartPartsInt[1]) || + (entryPartsInt[0] == chartPartsInt[0] && entryPartsInt[1] == chartPartsInt[1] && entryPartsInt[2] > chartPartsInt[2]) { + + logFields := map[string]interface{}{ + "oldVersion": chart.Version, + "newVersion": entry.Version, + "index": chart.IndexName, + "chart": chart.Name, + "newAppVersion": entry.AppVersion, + } + monitor.WithFields(logFields).Info("Tshere is a newer version") + } + } + } + + return nil +}