use new artifact hub cache (#67)

refactor cache code to use artifact hub cache api

README.md

@@ -29,6 +29,36 @@ $ helm install helm-exporter sstarcher/helm-exporter
 * `helm install helm-exporter sstarcher/helm-exporter` will install and metrics should scrape automatically if prometheus is running
 * If using Grafana you can use this Dashboard to have a list of what's running https://grafana.com/dashboards/9367
 
+## Configuration for Latest versions
+
+Two options exist for fetching the latest version information for a chart.
+* Direct fetch from a chart repository.  This will download the index for the registry and use that information to fetch the chart.
+
+```yaml
+# Helm configuration
+config:
+  helmRegistries:
+    overrideChartNames: {}
+      mysql: stable/test
+# If the helm charts are not stored on hub.helm.sh then a custom registry can be configured here.
+# Currently only index.yaml registry is supported (helm supports other registries as well)
+    override:
+      - registry:
+          url: "https://some.url/index.yaml" # Url to the index file
+        charts: # Chart names
+        - splunk
+        - falco-eks-audit-bridge
+```
+
+* Query https://artifacthub.io for the chart matching your chart name and only using the specified registries.  If no registry name is specified and multiple charts match from helm hub no version will be found and it will log a warning.
+```yaml
+# Helm configuration
+config:
+  helmRegistries:
+    registryNames:
+    - bitnami
+```
+
 # Metrics
 * http://host:9571/metrics
 

config/config.go

@@ -3,32 +3,27 @@ package config
 import (
 	log "github.com/sirupsen/logrus"
 
-	"github.com/sstarcher/helm-exporter/registries"
 	"github.com/knadh/koanf"
 	"github.com/knadh/koanf/parsers/yaml"
 	"github.com/knadh/koanf/providers/file"
+	"github.com/sstarcher/helm-exporter/registries"
 )
 
 // Config of the lcm application, normally loaded from the config file
 type Config struct {
 	HelmRegistries registries.HelmRegistries `koanf:"helmRegistries"`
 }
 
-// AppConfig is the config for the app which can be set trough cli and config
-type AppConfig struct {
-	ConfigFile string
-}
-
-// LoadConfiguration loads the configuration from file
-func LoadConfiguration(configFile string) Config {
-	log.WithField("configFile", configFile).Debug("Loading config file")
+// New returns a config object from the given file
+func New(fileName string) Config {
+	log.WithField("configFile", fileName).Debug("Loading config file")
 
 	var lcmConfig Config
 	k := koanf.New(".")
 
 	// load defaults
-	if len(configFile) > 0 {
-		if err := k.Load(file.Provider(configFile), yaml.Parser()); err != nil {
+	if len(fileName) > 0 {
+		if err := k.Load(file.Provider(fileName), yaml.Parser()); err != nil {
 			log.WithError(err).Fatal("Error loading config")
 		}
 		if err := k.Unmarshal("", &lcmConfig); err != nil {

example_conf.yaml

@@ -0,0 +1,3 @@
+helmRegistries:
+  registryNames:
+  - bitnami

helm/README.md

@@ -1,92 +1,67 @@
 # helm-exporter
 
-Installs [helm-exporter](https://github.com/sstarcher/helm-exporter) to export helm stats to prometheus.
+![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
 
-## TL;DR;
+Exporter for helm metrics
 
-```console
-$ helm install sstarcher/helm-exporter
-```
+**Homepage:** <https://github.com/sstarcher/helm-exporter>
 
-## Introduction
+## Maintainers
 
-This chart bootstraps a [helm-exporter](https://github.com/sstarcher/helm-exporter) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager.
+| Name | Email | Url |
+| ---- | ------ | --- |
+| sstarcher | [email protected] | https://shanestarcher.com |
 
-The chart comes with a ServiceMonitor for use with the [Prometheus Operator](https://github.com/helm/charts/tree/master/stable/prometheus-operator).
+## Source Code
 
-## Installing the Chart
+* <https://github.com/sstarcher/helm-exporter>
 
-To install the chart with the release name `my-release`:
-
-```console
-$ helm install sstarcher/helm-exporter --name my-release
-```
-
-The command deploys helm-exporter on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.
-
-## Uninstalling the Chart
-
-To uninstall/delete the `my-release` deployment:
-
-```console
-$ helm delete my-release
-```
-
-The command removes all the Kubernetes components associated with the chart and deletes the release.
-
-## Configuration
-
-The following table lists the configurable parameters of the helm-exporter chart that are in addition to values in a default helm chart.
+## Values
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
-| affinity | object | `{}` | Assign custom affinity rules for helm-exporter [https://kubernetes.io/docs/concepts/configuration/assign-pod-node/](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) |
-| config.helmRegistries.overrideChartNames | object | `{}` | Provide a name to substitute for the full names of resources e.g. `mysql: stable/mysql` |
-| config.helmRegistries.override[0].allowAllReleases | bool | `true` | This allows all semver versions, like release candidates or custom suffixes. |
-| config.helmRegistries.override[0].charts | list | `[]` | Chart names for the override (chart) registry/repo url |
-| config.helmRegistries.override[0].registry.url | string | `""` |  Url to the index file for a custom helm repo |
-| fullnameOverride | string | `""` | Provide a name to substitute for the full names of resources |
-| image.pullPolicy | string | `"Always"` | Image pull policy for the webhook integration jobs |
-| image.repository | string | `"sstarcher/helm-exporter"` | Repository to use for the webhook integration jobs |
-| imagePullSecrets | list | `[]` | Reference to one or more secrets to be used when pulling images |
-| infoMetric | bool | `true` | Specifies whether to generate the info metric. |
-| ingress.annotations | object | `{}` |  Annotations for the helm-exporter |
-| ingress.enabled | bool | `false` | If true, helm-exporter Ingress will be created |
-| ingress.hosts[0].host | string | `"chart-example.local"` | Ingress hostname |
-| ingress.hosts[0].paths | list | `[]` | Ingress paths |
-| ingress.tls | list | `[]` | Ingress TLS configuration (YAML) |
-| latestChartVersion | bool | `true` | Specifies whether to fetch the latest chart versions from repositories. |
-| nameOverride | string | `""` | Provide a name in place of helm-exporter |
-| namespaces | string | `""` | Specifies which namespaces to query for helm 3 metrics.  Defaults to all |
-| nodeSelector | object | `{}` | helm-exporter node selector [https://kubernetes.io/docs/user-guide/node-selection/](https://kubernetes.io/docs/user-guide/node-selection/ ) |
-| podAnnotations | object | `{}` | Annotations to add to the pod |
-| podSecurityContext | object | `{}` | SecurityContext for helm-exporter pod |
-| rbac.create | bool | `true` | Create RBAC resources |
-| replicaCount | int | `1` | Number of instances to deploy. |
-| resources | object | `{}` | Define resources requests and limits for single Pods. |
-| securityContext | object | `{}` | SecurityContext for a container |
-| service.port | int | `9571` | Port for Service to listen on. |
-| service.type | string | `"ClusterIP"` | Service type |
-| service.customScrapeEnableAnnotation | string | `""` | Custom annotation to enable scraping. If empty, `prometheus.io/scrape: "true"` will be used |
-| serviceAccount.create | bool | `true` | Create a default serviceaccount to use |
-| serviceAccount.name | string | `default` | Name for prometheus serviceaccount |
-| serviceMonitor.create | bool | `false` | Set to true if using the Prometheus Operator |
-| serviceMonitor.interval | string | `nil` | Interval at which metrics should be scraped |
-| serviceMonitor.namespace | string | `nil` | The namespace where the Prometheus Operator is deployed |
-| serviceMonitor.additionalLabels |object | `{}` | Additional labels to add to the ServiceMonitor	|
-| serviceMonitor.scrapeTimeout | string | `nil` | Scrape Timeout when the metrics endpoint is scraped |
-| timestampMetric | bool | `true` | Specifies whether to generate the timestamps metric. |
-| tolerations | list | `[]` | Tolerations for use with node taints [https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)|
-
-
-```console
-$ helm install my-release sstarcher/helm-exporter
-```
-
-Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example,
-
-```console
-$ helm install my-release sstarcher/helm-exporter -f values.yaml
-```
-
-> **Tip**: You can use the default [values.yaml](values.yaml)
+| affinity | object | `{}` |  |
+| config.helmRegistries.overrideChartNames | object | `{}` |  |
+| config.helmRegistries.override[0].allowAllReleases | bool | `true` |  |
+| config.helmRegistries.override[0].charts | list | `[]` |  |
+| config.helmRegistries.override[0].registry.url | string | `""` |  |
+| config.helmRegistries.registryNames | list | `[]` |  |
+| env | list | `[]` |  |
+| fullnameOverride | string | `""` |  |
+| image.pullPolicy | string | `"Always"` |  |
+| image.repository | string | `"sstarcher/helm-exporter"` |  |
+| image.tag | string | `""` |  |
+| imagePullSecrets | list | `[]` |  |
+| infoMetric | bool | `true` |  |
+| ingress.annotations | object | `{}` |  |
+| ingress.enabled | bool | `false` |  |
+| ingress.hosts[0].host | string | `"chart-example.local"` |  |
+| ingress.hosts[0].paths | list | `[]` |  |
+| ingress.tls | list | `[]` |  |
+| intervalDuration | int | `0` |  |
+| latestChartVersion | bool | `true` |  |
+| nameOverride | string | `""` |  |
+| namespaces | string | `""` |  |
+| nodeSelector | object | `{}` |  |
+| podAnnotations | object | `{}` |  |
+| podLabels | object | `{}` |  |
+| podSecurityContext | object | `{}` |  |
+| rbac.create | bool | `true` |  |
+| replicaCount | int | `1` |  |
+| resources | object | `{}` |  |
+| securityContext | object | `{}` |  |
+| service.annotations | object | `{}` |  |
+| service.port | int | `9571` |  |
+| service.type | string | `"ClusterIP"` |  |
+| serviceAccount.create | bool | `true` |  |
+| serviceAccount.name | string | `nil` |  |
+| serviceMonitor.additionalLabels | object | `{}` |  |
+| serviceMonitor.create | bool | `false` |  |
+| serviceMonitor.interval | string | `nil` |  |
+| serviceMonitor.namespace | string | `nil` |  |
+| serviceMonitor.scrapeTimeout | string | `nil` |  |
+| timestampMetric | bool | `true` |  |
+| tolerations | list | `[]` |  |
+
+----------------------------------------------
+Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0)

helm/values.yaml

@@ -106,3 +106,4 @@ config:
 #         - splunk
 #         - falco-eks-audit-bridge
         allowAllReleases: true # This allows all semver versions, like release candidates or custom suffixes. Default is false
+    registryNames: [] # Used to filter accepted registries from helm hub

main.go

@@ -9,10 +9,10 @@ import (
 	"time"
 
 	"github.com/sstarcher/helm-exporter/config"
-	"github.com/sstarcher/helm-exporter/registries"
 
 	cmap "github.com/orcaman/concurrent-map"
 
+	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
 
 	"os"
@@ -52,6 +52,8 @@ var (
 
 	fetchLatest = flag.Bool("latest-chart-version", true, "Attempt to fetch the latest chart version from registries. Defaults to true")
 
+	verbose = flag.Bool("verbose", false, "Enables debug logging. Defaults to false")
+
 	statusCodeMap = map[string]float64{
 		"unknown":          0,
 		"deployed":         1,
@@ -67,12 +69,6 @@ var (
 	prometheusHandler = promhttp.Handler()
 )
 
-func initFlags() config.AppConfig {
-	cliFlags := new(config.AppConfig)
-	cliFlags.ConfigFile = *configFile
-	return *cliFlags
-}
-
 func configureMetrics() (info *prometheus.GaugeVec, timestamp *prometheus.GaugeVec) {
 	if *infoMetric == true {
 		info = prometheus.NewGaugeVec(prometheus.GaugeOpts{
@@ -134,7 +130,7 @@ func runStats(config config.Config, info *prometheus.GaugeVec, timestamp *promet
 			latestVersion := ""
 
 			if *fetchLatest {
-				latestVersion = getLatestChartVersionFromHelm(item.Chart.Name(), config.HelmRegistries)
+				latestVersion = config.HelmRegistries.GetLatestVersionFromHelm(item.Chart.Name())
 			}
 
 			if info != nil {
@@ -147,12 +143,6 @@ func runStats(config config.Config, info *prometheus.GaugeVec, timestamp *promet
 	}
 }
 
-func getLatestChartVersionFromHelm(name string, helmRegistries registries.HelmRegistries) (version string) {
-	version = helmRegistries.GetLatestVersionFromHelm(name)
-	log.WithField("chart", name).Debugf("last chart repo version is  %v", version)
-	return
-}
-
 func runStatsPeriodically(interval time.Duration, config config.Config) {
 	for {
 		info, timestamp := configureMetrics()
@@ -244,9 +234,12 @@ func informer() {
 func main() {
 	flagenv.Parse()
 	flag.Parse()
-	cliFlags := initFlags()
-	config := config.LoadConfiguration(cliFlags.ConfigFile)
 
+	if *verbose == true {
+		logrus.SetLevel(logrus.DebugLevel)
+	}
+
+	config := config.New(*configFile)
 	runIntervalDuration, err := time.ParseDuration(*intervalDuration)
 	if err != nil {
 		log.Fatalf("invalid duration `%s`: %s", *intervalDuration, err)

registries/charts.go

@@ -2,14 +2,17 @@ package registries
 
 import (
 	"regexp"
+	"strings"
 
 	log "github.com/sirupsen/logrus"
+	"github.com/sstarcher/helm-exporter/versioning"
 )
 
 // HelmRegistries contains all the information regarding helm registries
 type HelmRegistries struct {
 	OverrideChartNames map[string]string      `koanf:"overrideChartNames"`
 	OverrideRegistries []HelmOverrideRegistry `koanf:"override"`
+	RegistryNames      []string               `koanf:"registryNames"`
 }
 
 // HelmOverrideRegistry contains information about which registry to use to fetch helm versions
@@ -26,6 +29,10 @@ type HelmRegistry struct {
 
 // GetLatestVersionFromHelm fetches the latest version of the helm chart
 func (h HelmRegistries) GetLatestVersionFromHelm(chart string) string {
+	if val, ok := h.OverrideChartNames[chart]; ok {
+		chart = val
+	}
+
 	log.WithField("chart", chart).Debug("Fetching version for chart")
 
 	for _, registry := range h.OverrideRegistries {
@@ -35,14 +42,45 @@ func (h HelmRegistries) GetLatestVersionFromHelm(chart string) string {
 				log.WithError(err).Fatal("Chart regexp not valid")
 			}
 			if match {
-				chartName := h.OverrideChartNames[chart]
-				if chartName == "" {
-					chartName = chart
+				return registry.getChartVersions(chart)
+			}
+		}
+	}
+
+	return h.fromArtifactHub(chart)
+}
+
+func (h HelmRegistries) fromArtifactHub(chart string) string {
+	logger := log.WithField("chart", chart)
+
+	charts := []hubChart{}
+	mu.Lock()
+	for _, val := range hubCache {
+		if val.Name == chart {
+			if len(h.RegistryNames) == 0 {
+				charts = append(charts, val)
+			} else {
+				for _, reg := range h.RegistryNames {
+					if val.Repository.Name == reg {
+						charts = append(charts, val)
+					}
 				}
-				return registry.getChartVersions(chartName)
 			}
 		}
 	}
+	mu.Unlock()
+
+	if len(charts) == 0 {
+		logger.Warnf("unable to find any charts matching %s on artifacthub", chart)
+		return versioning.Failure
+	} else if len(charts) > 1 {
+		regs := []string{}
+		for _, val := range charts {
+			regs = append(regs, val.Repository.Name)
+		}
+		logger.Warnf("faile to search chart info, found multiple registries that contain this chart %s on helm hub[%s], update the configuration to call out your chart registries", chart, strings.Join(regs, ", "))
+		return versioning.Multiple
+	}
 
-	return h.useHelmHub(chart)
+	return charts[0].Version
 }