[TT-9438] Add an option to enable Prometheus ServiceMonitor for /metrics endpoint

CHANGELOG.md

@@ -7,8 +7,10 @@
 - Added `disabled` feature in `validate_json` field of APIDefinition. 
 - Added a new Status resource called `latestTransaction` to the APIDefinition CRD which holds information about 
 last reconciliation. Now, any error can be observed there instead of checking Tyk Operator logs.
+- Added an option to enable `ServiceMonitor` in helm charts, in order Prometheus Operator to scrape `/metrics` endpoint.
 - Added `extraVolume` and `extraVolumeMounts` options to the helm chart. So, extra volumes can be mounted in Tyk Operator's manager pod, e.g., self-signed certificates.
 
+
 **Fixed**
 - Check if certificate already exists on tyk before uploading
 - Operator throwing lots of errors "the object has been modified; please apply your changes to the latest version and try again" while reconciling security policy

config/helm/kustomization.yaml

@@ -17,6 +17,7 @@ bases:
   - ../manager
   - ../webhook
   - ../certmanager
+  - ../prometheus
 
 patchesStrategicMerge:
   # Protect the /metrics endpoint by putting it behind auth.

config/prometheus/monitor.yaml

@@ -1,5 +1,5 @@
-
 # Prometheus Monitor Service (Metrics)
+TYK_OPERATOR_PROMETHEUS_SERVICEMONITOR_IF_STARTS: null # DO NOT CHANGE This will be replaced by `pre_helm.go` script.
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -18,3 +18,4 @@ spec:
   selector:
     matchLabels:
       control-plane: tyk-operator-controller-manager
+status: TYK_OPERATOR_PROMETHEUS_SERVICEMONITOR_IF_ENDS # DO NOT CHANGE This will be replaced by `pre_helm.go` script.

hack/helm/pre_helm.go

@@ -22,6 +22,8 @@ func main() {
 		{securityContext, securityContextTPL},
 		{imageRBAC, imageRBACTPL},
 		{nodeSelector, nodeSelectorTPL},
+		{serviceMonitorIfStarts, serviceMonitorIfStartsTPL},
+		{serviceMonitorIfEnds, serviceMonitorIfEndsTPL},
 		{extraVolume, extraVolumeTPL},
 		{extraVolumeMounts, extraVolumeMountsTPL},
 
@@ -130,6 +132,14 @@ const nodeSelectorTPL = `{{- if .Values.nodeSelector }}
 {{ toYaml .Values.nodeSelector | indent 8 }}
 {{- end }}`
 
+// Replaces hardcoded values for ServiceMonitor resource with helm templates.
+const (
+	serviceMonitorIfStarts    = `TYK_OPERATOR_PROMETHEUS_SERVICEMONITOR_IF_STARTS: null`
+	serviceMonitorIfStartsTPL = `{{ if .Values.serviceMonitor }}`
+	serviceMonitorIfEnds      = `status: TYK_OPERATOR_PROMETHEUS_SERVICEMONITOR_IF_ENDS`
+	serviceMonitorIfEndsTPL   = `{{ end }} `
+)
+
 const extraVolume = `- name: CONTROLLER_MANAGER_EXTRA_VOLUME`
 
 const extraVolumeTPL = `{{ if .Values.extraVolumes }}

helm/templates/all.yaml

@@ -523,6 +523,27 @@ metadata:
 spec:
   selfSigned: {}
 ---
+{{ if .Values.serviceMonitor }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    control-plane: tyk-operator-controller-manager
+  name: {{ include "tyk-operator-helm.fullname" . }}-controller-manager-metrics-monitor
+  namespace: {{ .Release.Namespace }}
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    path: /metrics
+    port: https
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
+  selector:
+    matchLabels:
+      control-plane: tyk-operator-controller-manager
+{{ end }} 
+---
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:

helm/values.yaml

@@ -61,3 +61,9 @@ rbac:
     pullPolicy: IfNotPresent
     tag: "v0.8.0"
   port: 8443
+
+# serviceMonitor decides deploying ServiceMonitor if Prometheus Operator scrapes /metrics.
+# By default it is set to false and set it to true to enable deploying Service Monitor.
+# Tyk Operator publishes a collection of performance metrics for each controller.
+# These metrics can be scraped by Prometheus Operator, by deploying ServiceMonitor resource.
+serviceMonitor: false