Merge pull request #141 from K-Phoen/new-alerts

Bootstrap new alerting support

Makefile

@@ -94,11 +94,22 @@ dev-env-remove: ## Remove the development k3d cluster.
 
 .PHONY: dev-env-provision
 dev-env-provision: ## Provision the development k3d cluster with useful tools (Grafana, Prometheus, Loki, ...).
+	# Helm repositories
 	helm repo add grafana https://grafana.github.io/helm-charts
+	helm repo add bitnami https://charts.bitnami.com/bitnami
+	helm repo add bitnami https://charts.bitnami.com/bitnami
+	helm repo add kiwigrid https://kiwigrid.github.io
 	helm repo update
+	# Grafana, loki, prometheus, ...
 	helm upgrade \
 		--install loki grafana/loki-stack \
-		--set grafana.enabled=true,grafana.image.tag=8.3.4,prometheus.enabled=true,prometheus.alertmanager.persistentVolume.enabled=false,prometheus.server.persistentVolume.enabled=false
+		--set grafana.enabled=true,grafana.image.tag=8.5.1,prometheus.enabled=true,prometheus.alertmanager.persistentVolume.enabled=false,prometheus.server.persistentVolume.enabled=false
+	# InfluxDB
+	helm upgrade --install influxdb bitnami/influxdb \
+		--set auth.enabled=false
+	# Graphite
+	helm upgrade --install graphite kiwigrid/graphite
+	# Rest
 	kubectl apply -f config/crd/bases
 	kubectl apply -f config/dev-env
 
@@ -107,6 +118,14 @@ dev-env-info: ## Print useful information for the dev environment (URL, credenti
 	@echo "==============="
 	@echo "Grafana available at http://$(DEV_GRAFANA_HOST):$(DEV_CLUSTER_PORT)"
 	@kubectl get secret loki-grafana -o go-template='{{range $$k,$$v := .data}}{{printf "%s: " $$k}}{{if not $$v}}{{$$v}}{{else}}{{$$v | base64decode}}{{end}}{{"\n"}}{{end}}'
+	@echo "==============="
+	@echo "InfluxDB available at: http://influxdb:8086"
+	@echo -n "admin password: "
+	@kubectl get secret --namespace default influxdb -o jsonpath="{.data.admin-user-password}" | base64 --decode
+	@echo ""
+	@echo "==============="
+	@echo "Graphite available at http://graphite:8080"
+
 
 .PHONY: dev-env-check-binaires
 dev-env-check-binaries: ## Check that the required binary are present.
@@ -203,4 +222,4 @@ echo "Downloading $(2)" ;\
 GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
 rm -rf $$TMP_DIR ;\
 }
-endef
+endef

README.md

@@ -34,8 +34,8 @@ kind: GrafanaDashboard
 metadata:
   name: example-dashboard
   namespace: monitoring
-
-folder: "Awesome folder"
+  annotations:
+    dark/folder: "Awesome folder"
 spec:
   title: Awesome dashboard
   shared_crosshair: true

api/v1/grafanadashboard_types.go

@@ -27,8 +27,9 @@ type GrafanaDashboard struct {
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
 	//+kubebuilder:pruning:PreserveUnknownFields
-	Spec   runtime.RawExtension `json:"spec"`
-	Folder string               `json:"folder"`
+	Spec runtime.RawExtension `json:"spec"`
+	//+kubebuilder:validation:Optional
+	Folder string `json:"folder"`
 
 	Status GrafanaDashboardStatus `json:"status,omitempty"`
 }

cmd/converter/cmd/tomanifest.go

@@ -13,7 +13,7 @@ func ToManifestCommand(logger *zap.Logger) *cobra.Command {
 	var options converter.K8SManifestOptions
 
 	var cmd = &cobra.Command{
-		Use:   "convert-k8s-manifest",
+		Use:   "convert-k8s-manifest name",
 		Args:  cobra.ExactArgs(1),
 		Short: "Converts a JSON dashboard into a k8s manifest",
 		Run: func(cmd *cobra.Command, args []string) {

config/crd/bases/k8s.kevingomez.fr_alertmanagers.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.8.0
   creationTimestamp: null
   name: alertmanagers.k8s.kevingomez.fr
 spec:

config/crd/bases/k8s.kevingomez.fr_apikeys.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.8.0
   creationTimestamp: null
   name: apikeys.k8s.kevingomez.fr
 spec:

config/crd/bases/k8s.kevingomez.fr_datasources.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.8.0
   creationTimestamp: null
   name: datasources.k8s.kevingomez.fr
 spec:

config/crd/bases/k8s.kevingomez.fr_grafanadashboards.yaml

@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.8.0
   creationTimestamp: null
   name: grafanadashboards.k8s.kevingomez.fr
 spec:
@@ -62,7 +61,6 @@ spec:
             - status
             type: object
         required:
-        - folder
         - spec
         type: object
     served: true

config/rbac/role.yaml

@@ -1,4 +1,3 @@
-
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

examples/dashboards/dark.yaml

@@ -2,11 +2,12 @@ apiVersion: k8s.kevingomez.fr/v1
 kind: GrafanaDashboard
 metadata:
   name: dark-reconciliations
+  annotations:
+    dark/folder: DARK
 
-folder: DARK
 spec:
   title: Dark operator reconciliations
-  editable: false
+  editable: true
   shared_crosshair: true
   tags: [dark, generated]
   auto_refresh: 5s
@@ -25,16 +26,36 @@ spec:
       - timeseries:
           title: GrafanaDashboards reconciliations status
           description: Reconciliation status of GrafanaDashboards manifests
+          datasource: Prometheus
           targets:
             - prometheus:
                 query: sum(increase(controller_runtime_reconcile_total{controller="grafanadashboard"}[$interval])) by (result)
                 legend: '{{ result }}'
+          alert:
+            summary: 'Alert summary'
+            description: "Alert msg"
+            evaluate_every: 3m
+            for: 6m
+            on_no_data: ok
+            on_execution_error: alerting
+            if:
+              - { avg: A, above: 0 }
+            targets:
+              - prometheus:
+                  ref: A
+                  query: sum(increase(controller_runtime_reconcile_total{controller="grafanadashboard"}[5m])) by (result)
+                  legend: "{{ result }}"
+                  lookback: 5m
+            tags:
+              owner: platform
+              service: service_name
+              og_priority: P4
 
       - timeseries:
           title: Datasources reconciliations status
           description: Reconciliation status of Datasources manifests
+          datasource: Prometheus
           targets:
             - prometheus:
                 query: sum(increase(controller_runtime_reconcile_total{controller="datasource"}[$interval])) by (result)
-                legend: '{{ result }}'
-
+                legend: '{{ result }}'

examples/dashboards/graphite-query.yaml

@@ -0,0 +1,34 @@
+apiVersion: k8s.kevingomez.fr/v1
+kind: GrafanaDashboard
+metadata:
+  name: dark-graphite-query
+  annotations:
+    dark/folder: DARK
+
+spec:
+  title: Dark with graphite query
+  editable: true
+  shared_crosshair: true
+  tags: [dark, graphite, generated]
+  auto_refresh: 5s
+  time: [now-1h, now]
+
+  rows:
+    - name: Overview
+      panels:
+      - timeseries:
+          title: Stats count
+          datasource: Graphite
+          targets:
+            - graphite:
+                query: 'aliasByMetric(stats_counts.*.*)'
+          alert:
+            summary: 'Some graphite-based alert'
+            description: "Very description, much wow"
+            targets:
+              - graphite:
+                  ref: A
+                  query: 'aliasByMetric(stats_counts.*.*)'
+                  lookback: 5m
+            if:
+              - { avg: A, above: 0 }

examples/dashboards/loki-query.yaml

@@ -0,0 +1,41 @@
+apiVersion: k8s.kevingomez.fr/v1
+kind: GrafanaDashboard
+metadata:
+  name: dark-loki-query
+  annotations:
+    dark/folder: DARK
+
+spec:
+  title: Dark with loki query
+  editable: true
+  shared_crosshair: true
+  tags: [dark, loki, generated]
+  auto_refresh: 5s
+  time: [now-1h, now]
+
+  rows:
+    - name: Overview
+      panels:
+      - timeseries:
+          title: Logs ingestion rate
+          datasource: Loki
+          span: 12
+          targets:
+            - loki:
+                query: 'sum(rate({app="loki"}[$__interval])) by (app)'
+                legend: '{{ app }}'
+          alert:
+            summary: 'No logs ingested on loki'
+            targets:
+              - loki:
+                  ref: A
+                  query: 'count_over_time({app="loki"}[5m])'
+                  lookback: 5m
+            if:
+              - { avg: A, below: 0 }
+      - logs:
+          title: Loki logs explorer
+          datasource: Loki
+          span: 12
+          targets:
+            - loki: { query: "{app=\"loki\"}" }

examples/dashboards/stackdriver-query.yaml

@@ -0,0 +1,48 @@
+apiVersion: k8s.kevingomez.fr/v1
+kind: GrafanaDashboard
+metadata:
+  name: dark-stackdriver-query
+  annotations:
+    dark/folder: DARK
+
+spec:
+  title: Dark with stackdriver query
+  editable: true
+  shared_crosshair: true
+  tags: [dark, stackdriver, generated]
+  auto_refresh: 5s
+  time: [now-1h, now]
+
+  rows:
+    - name: Overview
+      panels:
+      - timeseries:
+          title: CPU utilization per database
+          datasource: Stackdriver
+          axis: { unit: percentunit }
+          targets:
+            - stackdriver:
+                type: gauge
+                metric: cloudsql.googleapis.com/database/cpu/utilization
+                group_by: [resource.label.database_id]
+                legend: '{{ resource.label.database_id }}'
+                aggregation: mean
+                alignment:
+                  method: mean
+                  period: stackdriver-auto
+          alert:
+            summary: 'High CPU usage'
+            targets:
+              - stackdriver:
+                  ref: A
+                  lookback: 5m
+                  type: gauge
+                  metric: cloudsql.googleapis.com/database/cpu/utilization
+                  group_by: [ resource.label.database_id ]
+                  legend: '{{ resource.label.database_id }}'
+                  aggregation: mean
+                  alignment:
+                    method: mean
+                    period: stackdriver-auto
+            if:
+              - { avg: A, above: 0.75 }

go.mod

@@ -3,17 +3,17 @@ module github.com/K-Phoen/dark
 go 1.17
 
 require (
-	github.com/K-Phoen/grabana v0.20.11
-	github.com/K-Phoen/sdk v0.8.4
-	github.com/go-logr/logr v1.2.3
+	github.com/K-Phoen/grabana v0.21.6
+	github.com/K-Phoen/sdk v0.11.0
+	github.com/go-logr/logr v1.2.2
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.17.0
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.11.0
 	github.com/stretchr/testify v1.7.1
 	go.uber.org/zap v1.21.0
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	gopkg.in/yaml.v3 v3.0.0
 	k8s.io/api v0.24.0
 	k8s.io/apimachinery v0.24.0
 	k8s.io/client-go v0.24.0
@@ -67,10 +67,10 @@ require (
 	github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.11.0 // indirect
+	github.com/prometheus/client_golang v1.12.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.32.0 // indirect
-	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/prometheus/common v0.34.0 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/spf13/afero v1.8.2 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect