Add service monitor for flyte admin and propeller service (flyteorg#4427

) * Add service monitor for flyte admin and propeller service Signed-off-by: vraiyani <[email protected]> * make interval/scrapeTimeout configurable for service monitors Signed-off-by: vraiyani <[email protected]> --------- Signed-off-by: vraiyani <[email protected]> Co-authored-by: David Espejo <[email protected]>
Tom-Newton · Nov 20, 2023 · 568e686 · 568e686
1 parent 956b24e
commit 568e686
Show file tree

Hide file tree

Showing 9 changed files with 110 additions and 1 deletion.
diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md
@@ -176,6 +176,11 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.serviceAccount.create | bool | `true` | Should a service account be created for flyteadmin |
 | flyteadmin.serviceAccount.createClusterRole | bool | `true` | Should a ClusterRole be created for Flyteadmin |
 | flyteadmin.serviceAccount.imagePullSecrets | list | `[]` | ImagePullSecrets to automatically assign to the service account |
+| flyteadmin.serviceMonitor | object | `{"enabled":false,"interval":"60s","labels":{},"scrapeTimeout":"30s"}` | Settings for flyteadmin service monitor |
+| flyteadmin.serviceMonitor.enabled | bool | `false` | If enabled create the flyteadmin service monitor |
+| flyteadmin.serviceMonitor.interval | string | `"60s"` | Sets the interval at which metrics will be scraped by prometheus |
+| flyteadmin.serviceMonitor.labels | object | `{}` | Sets the labels for the service monitor which are required by the prometheus to auto-detect the service monitor and start scrapping the metrics |
+| flyteadmin.serviceMonitor.scrapeTimeout | string | `"30s"` | Sets the timeout after which request to scrape metrics will time out |
 | flyteadmin.tolerations | list | `[]` | tolerations for Flyteadmin deployment |
 | flyteagent.enabled | bool | `false` |  |
 | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment |
@@ -211,10 +216,17 @@ helm install gateway bitnami/contour -n flyte
 | flytepropeller.priorityClassName | string | `""` | Sets priorityClassName for propeller pod(s). |
 | flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment |
 | flytepropeller.resources | object | `{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"100Mi"}}` | Default resources requests and limits for Flytepropeller deployment |
+| flytepropeller.service | object | `{"enabled":false}` | Settings for flytepropeller service |
+| flytepropeller.service.enabled | bool | `false` | If enabled create the flytepropeller service |
 | flytepropeller.serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":[]}` | Configuration for service accounts for FlytePropeller |
 | flytepropeller.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to FlytePropeller pods |
 | flytepropeller.serviceAccount.create | bool | `true` | Should a service account be created for FlytePropeller |
 | flytepropeller.serviceAccount.imagePullSecrets | list | `[]` | ImagePullSecrets to automatically assign to the service account |
+| flytepropeller.serviceMonitor | object | `{"enabled":false,"interval":"60s","labels":{},"scrapeTimeout":"30s"}` | Settings for flytepropeller service monitor |
+| flytepropeller.serviceMonitor.enabled | bool | `false` | If enabled create the flyetepropeller service monitor |
+| flytepropeller.serviceMonitor.interval | string | `"60s"` | Sets the interval at which metrics will be scraped by prometheus |
+| flytepropeller.serviceMonitor.labels | object | `{}` | Sets the labels for the service monitor which are required by the prometheus to auto-detect the service monitor and start scrapping the metrics |
+| flytepropeller.serviceMonitor.scrapeTimeout | string | `"30s"` | Sets the timeout after which request to scrape metrics will time out |
 | flytepropeller.terminationMessagePolicy | string | `"FallbackToLogsOnError"` | Error reporting |
 | flytepropeller.tolerations | list | `[]` | tolerations for Flytepropeller deployment |
 | flytescheduler.additionalContainers | list | `[]` | Appends additional containers to the deployment spec. May include template values. |

diff --git a/charts/flyte-core/templates/admin/service-monitor.yaml b/charts/flyte-core/templates/admin/service-monitor.yaml
@@ -0,0 +1,19 @@
+{{- if and .Values.flyteadmin.serviceMonitor.enabled .Values.flyteadmin.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "flyteadmin.name" . }}
+  namespace: {{ template "flyte.namespace" . }}
+  labels:
+  {{- with .Values.flyteadmin.serviceMonitor.labels }}
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+    - interval: {{ .Values.flyteadmin.serviceMonitor.interval }}
+      port: http-metrics
+      path: /metrics
+      scrapeTimeout: {{ .Values.flyteadmin.serviceMonitor.scrapeTimeout }}
+  selector:
+    matchLabels: {{ include "flyteadmin.selectorLabels" . | nindent 6 }}
+{{- end }}
diff --git a/charts/flyte-core/templates/propeller/service-monitor.yaml b/charts/flyte-core/templates/propeller/service-monitor.yaml
@@ -0,0 +1,19 @@
+{{- if and .Values.flytepropeller.service.enabled .Values.flytepropeller.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  namespace: {{ template "flyte.namespace" . }}
+  name: {{ template "flytepropeller.name" . }}
+  labels:
+  {{- with .Values.flytepropeller.serviceMonitor.labels }}
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+    - interval: {{ .Values.flytepropeller.serviceMonitor.interval }}
+      port: http-metrics
+      path: /metrics
+      scrapeTimeout: {{ .Values.flytepropeller.serviceMonitor.scrapeTimeout }}
+  selector:
+    matchLabels: {{ include "flytepropeller.selectorLabels" . | nindent 6 }}
+{{- end }}
diff --git a/charts/flyte-core/templates/propeller/service.yaml b/charts/flyte-core/templates/propeller/service.yaml
@@ -0,0 +1,15 @@
+{{- if and .Values.flytepropeller.enabled .Values.flytepropeller.service.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: {{ template "flyte.namespace" . }}
+  name: {{ template "flytepropeller.name" . }}
+  labels: {{ include "flytepropeller.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - name: http-metrics
+      protocol: TCP
+      port: 10254
+  selector: {{ include "flytepropeller.selectorLabels" . | nindent 4 }}
+{{- end }}
diff --git a/charts/flyte-core/values-eks.yaml b/charts/flyte-core/values-eks.yaml
@@ -23,6 +23,9 @@ flyteadmin:
     annotations:
       eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.userSettings.accountNumber }}:role/iam-role-flyte
 
+  serviceMonitor:
+    enabled: false
+
   resources:
     limits:
       ephemeral-storage: 200Mi
@@ -78,6 +81,10 @@ datacatalog:
 flytepropeller:
   replicaCount: 2
   manager: false
+  serviceMonitor:
+    enabled: false
+  service:
+    enabled: false
   serviceAccount:
     # -- If the service account is created by you, make this false
     create: true

diff --git a/charts/flyte-core/values-gcp.yaml b/charts/flyte-core/values-gcp.yaml
@@ -14,6 +14,8 @@ userSettings:
 
 flyteadmin:
   replicaCount: 1
+  serviceMonitor:
+    enabled: false
   serviceAccount:
     # -- If the service account is created by you, make this false, else a new service account will be created and the flyteadmin role will be added
     # you can change the name of this role
@@ -82,6 +84,10 @@ datacatalog:
 flytepropeller:
   replicaCount: 1
   manager: false
+  serviceMonitor:
+    enabled: false
+  service:
+    enabled: false
   serviceAccount:
     # -- If the service account is created by you, make this false, else a new service account will be created and the iam-role-flyte will be added
     # you can change the name of this role

diff --git a/charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml b/charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml
@@ -11,6 +11,8 @@
 
 flyteadmin:
   enabled: true
+  serviceMonitor:
+    enabled: false
   # -- Replicas count for Flyteadmin deployment
   replicaCount: 1
   image:

diff --git a/charts/flyte-core/values-sandbox.yaml b/charts/flyte-core/values-sandbox.yaml
@@ -1,5 +1,7 @@
 
 flyteadmin:
+  serviceMonitor:
+    enabled: false
   service:
     annotations:
       projectcontour.io/upstream-protocol.h2c: grpc

diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml
@@ -110,6 +110,17 @@ flyteadmin:
   # -- Sets priorityClassName for flyteadmin pod(s).
   priorityClassName: ""
 
+  # -- Settings for flyteadmin service monitor
+  serviceMonitor:
+    # -- If enabled create the flyteadmin service monitor
+    enabled: false
+    # -- Sets the interval at which metrics will be scraped by prometheus
+    interval: 60s
+    # -- Sets the timeout after which request to scrape metrics will time out
+    scrapeTimeout: 30s
+    # -- Sets the labels for the service monitor which are required by the
+    # prometheus to auto-detect the service monitor and start scrapping the metrics
+    labels: {}
 #
 # FLYTESCHEDULER SETTINGS
 #
@@ -290,6 +301,22 @@ flytepropeller:
   # -- Sets priorityClassName for propeller pod(s).
   priorityClassName: ""
 
+  # -- Settings for flytepropeller service
+  service:
+    # -- If enabled create the flytepropeller service
+    enabled: false
+
+  # -- Settings for flytepropeller service monitor
+  serviceMonitor:
+    # -- If enabled create the flyetepropeller service monitor
+    enabled: false
+    # -- Sets the labels for the service monitor which are required by the
+    # prometheus to auto-detect the service monitor and start scrapping the metrics
+    labels: {}
+    # -- Sets the interval at which metrics will be scraped by prometheus
+    interval: 60s
+    # -- Sets the timeout after which request to scrape metrics will time out
+    scrapeTimeout: 30s
 #
 # FLYTECONSOLE SETTINGS
 #
@@ -895,4 +922,4 @@ databricks:
       databricks:
         entrypointFile: dbfs:///FileStore/tables/entrypoint.py
         # Databricks account
-        databricksInstance: dbc-a53b7a3c-614c
+        databricksInstance: dbc-a53b7a3c-614c