Add monitoring using prometheus-stack

charts/harmony-chart/Chart.lock

@@ -5,20 +5,23 @@ dependencies:
 - name: cert-manager
   repository: https://charts.jetstack.io
   version: v1.11.0
-- name: elasticsearch
-  repository: https://helm.elastic.co
-  version: 7.17.3
 - name: metrics-server
   repository: https://kubernetes-sigs.github.io/metrics-server/
   version: 3.8.4
 - name: vertical-pod-autoscaler
   repository: https://cowboysysop.github.io/charts/
   version: 6.0.3
+- name: elasticsearch
+  repository: https://helm.elastic.co
+  version: 7.17.3
 - name: opensearch
   repository: https://opensearch-project.github.io/helm-charts
   version: 2.13.3
 - name: karpenter
   repository: oci://public.ecr.aws/karpenter
   version: v0.29.2
-digest: sha256:453b9f734e2d770948d3cbd36529d98da284b96de051581ea8d11a3c05e7a78e
-generated: "2023-10-03T10:52:43.453442762-05:00"
+- name: kube-prometheus-stack
+  repository: https://prometheus-community.github.io/helm-charts
+  version: 56.6.2
+digest: sha256:8d3ab6c27a75b2ab6106164e38ecfaeaecfb329abc3526c25565f4d6fa6d4ed5
+generated: "2024-02-05T14:59:50.438116+04:00"

charts/harmony-chart/Chart.yaml

@@ -5,7 +5,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes to the chart and its
 # templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.0
+version: 0.4.0
 # This is the version number of the application being deployed. This version number should be incremented each time you
 # make changes to the application. Versions are not expected to follow Semantic Versioning. They should reflect the
 # version the application is using. It is recommended to use it with quotes.
@@ -26,23 +26,23 @@ dependencies:
   repository: https://charts.jetstack.io
   condition: cert-manager.enabled
 
-- name: elasticsearch
-  version: "7.17.3"
-  condition: elasticsearch.enabled
-  repository: https://helm.elastic.co
-
 - name: metrics-server
+  alias: metricsserver
   version: "~3.8.3"
   repository: https://kubernetes-sigs.github.io/metrics-server/
-  alias: metricsserver
   condition: metricsserver.enabled
 
 - name: vertical-pod-autoscaler
+  alias: vpa
   version: "~6.0.3"
   repository: https://cowboysysop.github.io/charts/
-  alias: vpa
   condition: vpa.enabled
 
+- name: elasticsearch
+  version: "7.17.3"
+  condition: elasticsearch.enabled
+  repository: https://helm.elastic.co
+
 - name: opensearch
   version: "2.13.3"
   condition: opensearch.enabled
@@ -52,3 +52,9 @@ dependencies:
   version: "v0.29.2"
   repository: oci://public.ecr.aws/karpenter
   condition: karpenter.enabled
+
+- name: kube-prometheus-stack
+  alias: prometheusstack
+  version: "56.6.2"
+  condition: prometheusstack.enabled
+  repository: https://prometheus-community.github.io/helm-charts

charts/harmony-chart/templates/NOTES.txt

@@ -7,9 +7,14 @@ that the load balancer is working. First, get its external IP using
 
     kubectl get svc -n {{ .Release.Namespace }} harmony-ingress-nginx-controller
 
-Next, go to http://the.external.ip.shown/cluster-echo-test and make sure you get
+Next, go to http://{{ .Values.clusterDomain }}/cluster-echo-test and make sure you get
 a JSON response.
 
+{{ if and .Values.prometheusstack.grafana.enabled .Values.prometheusstack.grafana.ingress.enabled }}
+Grafana shipped with the default admin user password as a bug prevents
+changing it. Since is enabled on the cluster and exposed to the internet.
+Please make sure you update the default admin user password!
+{{- end }}
 
 
 

charts/harmony-chart/templates/echo.yaml

@@ -47,7 +47,8 @@ metadata:
 spec:
   ingressClassName: {{ (index .Values "ingress-nginx" "controller" "ingressClass") }}
   rules:
-  - http:
+  - host: {{ .Values.clusterDomain }}
+    http:
       paths:
       - path: /cluster-echo-test
         pathType: Prefix

charts/harmony-chart/values.yaml

@@ -2,6 +2,8 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
+clusterDomain: "*"
+
 ingress-nginx:
   # Use ingress-nginx as a default controller.
   enabled: true
@@ -14,6 +16,23 @@ cert-manager:
     # certificates.
   email: ""
 
+# Configuration for the metrics server chart
+metricsserver:
+   # Control the chart inclusion
+   enabled: false
+   # See https://github.com/kubernetes-sigs/metrics-server/blob/master/charts/metrics-server/values.yaml
+   # for all available options
+   replicas: 1
+
+# Configuration for the Vertical Pod Autoscaler chart
+vpa:
+   # Control the chart inclusion
+   enabled: false
+   # See https://github.com/cowboysysop/charts/blob/master/charts/vertical-pod-autoscaler/values.yaml
+   # for all available options
+   admissionController:
+      replicaCount: 1
+
 # Multi-tenant ElasticSearch
 elasticsearch:
   enabled: false
@@ -66,22 +85,6 @@ elasticsearch:
       xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certs/ca.crt
       xpack.security.transport.ssl.verification_mode: certificate
 
-# Configuration for the metrics server chart
-metricsserver:
-   # Control the chart inclusion
-   enabled: false
-   # See https://github.com/kubernetes-sigs/metrics-server/blob/master/charts/metrics-server/values.yaml
-   # for all available options
-   replicas: 1
-# Configuration for the Vertical Pod Autoscaler chart
-vpa:
-   # Control the chart inclusion
-   enabled: false
-   # See https://github.com/cowboysysop/charts/blob/master/charts/vertical-pod-autoscaler/values.yaml
-   # for all available options
-   admissionController:
-      replicaCount: 1
-
 # Multi-tenant OpenSearch
 opensearch:
   enabled: false
@@ -99,7 +102,6 @@ opensearch:
   # K8S_HARMONY_USE_SHARED_OPENSEARCH: true
   # HARMONY_SEARCH_HTTP_AUTH: "username:actual_password"
 
-
   # # This secret will contain the ssl certificates.
   secretMounts:
     - name: opensearch-certificates
@@ -235,3 +237,77 @@ karpenter:
   # Node template reference. More details in https://karpenter.sh/docs/concepts/node-templates/
   nodeTemplate:
     name: "default"
+
+# Prometheus stack
+#   
+# If no storage is defined, the Prometheus and Grafana data is stored on
+# empheral storage. You can find more information about storages in the
+# Prometheus operator user guides: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md
+# 
+# - To make Grafana persistent, you may want to set Grafana storage as shown in https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml#L999-L1009
+# - To set Prometheus storage, you can see the example storage spec at https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml#L3557-L3570
+prometheusstack:
+  enabled: false
+
+  kubeStateMetrics:
+    enabled: true
+
+  nodeExporter:
+    enabled: true
+
+  prometheus:
+    enabled: true
+
+    prometheusSpec:
+      resources:
+        requests:
+          cpu: "200m"
+          memory: "450Mi"
+
+  alertmanager:
+    enabled: true
+
+    alertmanagerSpec:
+      resources:
+        requests:
+          cpu: "100m"
+          memory: "50Mi"
+
+  # Admin password is not pre-generated, because it is not picked up by the
+  # grafana pod yet -- this is a bug on their end. For more information, visit:
+  # https://github.com/prometheus-community/helm-charts/issues/3679
+  grafana:
+    enabled: false
+
+    ingress:
+      enabled: false
+
+    dashboardProviders:
+      dashboardproviders.yaml:
+        apiVersion: 1
+        providers:
+          - name: 'default'
+            orgId: 1
+            folder: ''
+            type: file
+            disableDeletion: false
+            editable: true
+            options:
+              path: /var/lib/grafana/dashboards/default
+
+    dashboards:
+      default:
+        kubernetes-views-global:
+          # url: https://grafana.com/api/dashboards/15757/revisions/31/download
+          gnetId: 15757
+          revision: 31
+          datasource: Prometheus
+
+    grafana.ini:
+      dashboards:
+        default_home_dashboard_path: /var/lib/grafana/dashboards/default/kubernetes-views-global.json
+
+    resources:
+      requests:
+        cpu: 200m
+        memory: 256Mi

values-example.yaml

@@ -1,7 +1,10 @@
+clusterDomain: "example.com"
+
 ingress-nginx:
   controller:
     config:
       proxy-body-size: 100m
+
 cert-manager:
   # Set your email address here so auto-generated HTTPS certs will work:
   email: "[email protected]"
@@ -11,8 +14,31 @@ elasticsearch:
 
 metricsserver:
   enabled: false
+
 vpa:
   enabled: false
 
 opensearch:
   enabled: false
+
+prometheusstack:
+  enabled: false
+
+  grafana:
+    enabled: false
+
+    ingress:
+      enabled: true
+      ingressClassName: nginx
+      annotations:
+        cert-manager.io/cluster-issuer: "harmony-letsencrypt-global"
+        nginx.ingress.kubernetes.io/ssl-redirect: "true"
+      hosts:
+        - grafana.example.com
+      tls:
+        - secretName: promstack-ingress-tls
+          hosts:
+            - grafana.example.com
+
+  # alertmanager:
+  #   config: {}  # Set it using `--set-file prometheusstack.alertmanager.config=<path-to-file>`

values-minikube.yaml

@@ -2,6 +2,8 @@
 cert-manager:
    enabled: false
 
+clusterDomain: "example.local"
+
 elasticsearch:
    enabled: false
 
@@ -22,3 +24,6 @@ opensearch:
 
    persistence:
      size: 8Gi
+
+prometheusstack:
+  enabled: false