Merge branch 'master' into feature/open-telemetry

Signed-off-by: Daniel Rammer <[email protected]>

charts/flyte-core/README.md

@@ -176,6 +176,11 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.serviceAccount.create | bool | `true` | Should a service account be created for flyteadmin |
 | flyteadmin.serviceAccount.createClusterRole | bool | `true` | Should a ClusterRole be created for Flyteadmin |
 | flyteadmin.serviceAccount.imagePullSecrets | list | `[]` | ImagePullSecrets to automatically assign to the service account |
+| flyteadmin.serviceMonitor | object | `{"enabled":false,"interval":"60s","labels":{},"scrapeTimeout":"30s"}` | Settings for flyteadmin service monitor |
+| flyteadmin.serviceMonitor.enabled | bool | `false` | If enabled create the flyteadmin service monitor |
+| flyteadmin.serviceMonitor.interval | string | `"60s"` | Sets the interval at which metrics will be scraped by prometheus |
+| flyteadmin.serviceMonitor.labels | object | `{}` | Sets the labels for the service monitor which are required by the prometheus to auto-detect the service monitor and start scrapping the metrics |
+| flyteadmin.serviceMonitor.scrapeTimeout | string | `"30s"` | Sets the timeout after which request to scrape metrics will time out |
 | flyteadmin.tolerations | list | `[]` | tolerations for Flyteadmin deployment |
 | flyteagent.enabled | bool | `false` |  |
 | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment |
@@ -211,10 +216,17 @@ helm install gateway bitnami/contour -n flyte
 | flytepropeller.priorityClassName | string | `""` | Sets priorityClassName for propeller pod(s). |
 | flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment |
 | flytepropeller.resources | object | `{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"100Mi"}}` | Default resources requests and limits for Flytepropeller deployment |
+| flytepropeller.service | object | `{"enabled":false}` | Settings for flytepropeller service |
+| flytepropeller.service.enabled | bool | `false` | If enabled create the flytepropeller service |
 | flytepropeller.serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":[]}` | Configuration for service accounts for FlytePropeller |
 | flytepropeller.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to FlytePropeller pods |
 | flytepropeller.serviceAccount.create | bool | `true` | Should a service account be created for FlytePropeller |
 | flytepropeller.serviceAccount.imagePullSecrets | list | `[]` | ImagePullSecrets to automatically assign to the service account |
+| flytepropeller.serviceMonitor | object | `{"enabled":false,"interval":"60s","labels":{},"scrapeTimeout":"30s"}` | Settings for flytepropeller service monitor |
+| flytepropeller.serviceMonitor.enabled | bool | `false` | If enabled create the flyetepropeller service monitor |
+| flytepropeller.serviceMonitor.interval | string | `"60s"` | Sets the interval at which metrics will be scraped by prometheus |
+| flytepropeller.serviceMonitor.labels | object | `{}` | Sets the labels for the service monitor which are required by the prometheus to auto-detect the service monitor and start scrapping the metrics |
+| flytepropeller.serviceMonitor.scrapeTimeout | string | `"30s"` | Sets the timeout after which request to scrape metrics will time out |
 | flytepropeller.terminationMessagePolicy | string | `"FallbackToLogsOnError"` | Error reporting |
 | flytepropeller.tolerations | list | `[]` | tolerations for Flytepropeller deployment |
 | flytescheduler.additionalContainers | list | `[]` | Appends additional containers to the deployment spec. May include template values. |

charts/flyte-core/templates/admin/service-monitor.yaml

@@ -0,0 +1,19 @@
+{{- if and .Values.flyteadmin.serviceMonitor.enabled .Values.flyteadmin.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "flyteadmin.name" . }}
+  namespace: {{ template "flyte.namespace" . }}
+  labels:
+  {{- with .Values.flyteadmin.serviceMonitor.labels }}
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+    - interval: {{ .Values.flyteadmin.serviceMonitor.interval }}
+      port: http-metrics
+      path: /metrics
+      scrapeTimeout: {{ .Values.flyteadmin.serviceMonitor.scrapeTimeout }}
+  selector:
+    matchLabels: {{ include "flyteadmin.selectorLabels" . | nindent 6 }}
+{{- end }}

charts/flyte-core/templates/propeller/service-monitor.yaml

@@ -0,0 +1,19 @@
+{{- if and .Values.flytepropeller.service.enabled .Values.flytepropeller.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  namespace: {{ template "flyte.namespace" . }}
+  name: {{ template "flytepropeller.name" . }}
+  labels:
+  {{- with .Values.flytepropeller.serviceMonitor.labels }}
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+    - interval: {{ .Values.flytepropeller.serviceMonitor.interval }}
+      port: http-metrics
+      path: /metrics
+      scrapeTimeout: {{ .Values.flytepropeller.serviceMonitor.scrapeTimeout }}
+  selector:
+    matchLabels: {{ include "flytepropeller.selectorLabels" . | nindent 6 }}
+{{- end }}

charts/flyte-core/templates/propeller/service.yaml

@@ -0,0 +1,15 @@
+{{- if and .Values.flytepropeller.enabled .Values.flytepropeller.service.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: {{ template "flyte.namespace" . }}
+  name: {{ template "flytepropeller.name" . }}
+  labels: {{ include "flytepropeller.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - name: http-metrics
+      protocol: TCP
+      port: 10254
+  selector: {{ include "flytepropeller.selectorLabels" . | nindent 4 }}
+{{- end }}

charts/flyte-core/values-eks.yaml

@@ -23,6 +23,9 @@ flyteadmin:
     annotations:
       eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.userSettings.accountNumber }}:role/iam-role-flyte
 
+  serviceMonitor:
+    enabled: false
+
   resources:
     limits:
       ephemeral-storage: 200Mi
@@ -78,6 +81,10 @@ datacatalog:
 flytepropeller:
   replicaCount: 2
   manager: false
+  serviceMonitor:
+    enabled: false
+  service:
+    enabled: false
   serviceAccount:
     # -- If the service account is created by you, make this false
     create: true

charts/flyte-core/values-gcp.yaml

@@ -14,6 +14,8 @@ userSettings:
 
 flyteadmin:
   replicaCount: 1
+  serviceMonitor:
+    enabled: false
   serviceAccount:
     # -- If the service account is created by you, make this false, else a new service account will be created and the flyteadmin role will be added
     # you can change the name of this role
@@ -82,6 +84,10 @@ datacatalog:
 flytepropeller:
   replicaCount: 1
   manager: false
+  serviceMonitor:
+    enabled: false
+  service:
+    enabled: false
   serviceAccount:
     # -- If the service account is created by you, make this false, else a new service account will be created and the iam-role-flyte will be added
     # you can change the name of this role

charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml

@@ -11,6 +11,8 @@
 
 flyteadmin:
   enabled: true
+  serviceMonitor:
+    enabled: false
   # -- Replicas count for Flyteadmin deployment
   replicaCount: 1
   image:

charts/flyte-core/values-sandbox.yaml

@@ -1,5 +1,7 @@
 
 flyteadmin:
+  serviceMonitor:
+    enabled: false
   service:
     annotations:
       projectcontour.io/upstream-protocol.h2c: grpc

charts/flyte-core/values.yaml

@@ -110,6 +110,17 @@ flyteadmin:
   # -- Sets priorityClassName for flyteadmin pod(s).
   priorityClassName: ""
 
+  # -- Settings for flyteadmin service monitor
+  serviceMonitor:
+    # -- If enabled create the flyteadmin service monitor
+    enabled: false
+    # -- Sets the interval at which metrics will be scraped by prometheus
+    interval: 60s
+    # -- Sets the timeout after which request to scrape metrics will time out
+    scrapeTimeout: 30s
+    # -- Sets the labels for the service monitor which are required by the
+    # prometheus to auto-detect the service monitor and start scrapping the metrics
+    labels: {}
 #
 # FLYTESCHEDULER SETTINGS
 #
@@ -290,6 +301,22 @@ flytepropeller:
   # -- Sets priorityClassName for propeller pod(s).
   priorityClassName: ""
 
+  # -- Settings for flytepropeller service
+  service:
+    # -- If enabled create the flytepropeller service
+    enabled: false
+
+  # -- Settings for flytepropeller service monitor
+  serviceMonitor:
+    # -- If enabled create the flyetepropeller service monitor
+    enabled: false
+    # -- Sets the labels for the service monitor which are required by the
+    # prometheus to auto-detect the service monitor and start scrapping the metrics
+    labels: {}
+    # -- Sets the interval at which metrics will be scraped by prometheus
+    interval: 60s
+    # -- Sets the timeout after which request to scrape metrics will time out
+    scrapeTimeout: 30s
 #
 # FLYTECONSOLE SETTINGS
 #
@@ -895,4 +922,4 @@ databricks:
       databricks:
         entrypointFile: dbfs:///FileStore/tables/entrypoint.py
         # Databricks account
-        databricksInstance: dbc-a53b7a3c-614c
+        databricksInstance: dbc-a53b7a3c-614c

flyteplugins/go/tasks/plugins/k8s/kfoperators/tensorflow/tensorflow.go

@@ -217,16 +217,24 @@ func (tensorflowOperatorResourceHandler) BuildResource(ctx context.Context, task
 	return job, nil
 }
 
+func getReplicaCount(specs map[commonOp.ReplicaType]*commonOp.ReplicaSpec, replicaType commonOp.ReplicaType) *int32 {
+	if spec, ok := specs[replicaType]; ok && spec.Replicas != nil {
+		return spec.Replicas
+	}
+
+	return new(int32) // return 0 as default value
+}
+
 // Analyses the k8s resource and reports the status as TaskPhase. This call is expected to be relatively fast,
 // any operations that might take a long time (limits are configured system-wide) should be offloaded to the
 // background.
 func (tensorflowOperatorResourceHandler) GetTaskPhase(_ context.Context, pluginContext k8s.PluginContext, resource client.Object) (pluginsCore.PhaseInfo, error) {
 	app := resource.(*kubeflowv1.TFJob)
 
-	workersCount := app.Spec.TFReplicaSpecs[kubeflowv1.TFJobReplicaTypeWorker].Replicas
-	psReplicasCount := app.Spec.TFReplicaSpecs[kubeflowv1.TFJobReplicaTypePS].Replicas
-	chiefCount := app.Spec.TFReplicaSpecs[kubeflowv1.TFJobReplicaTypeChief].Replicas
-	evaluatorReplicasCount := app.Spec.TFReplicaSpecs[kubeflowv1.TFJobReplicaTypeEval].Replicas
+	workersCount := getReplicaCount(app.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypeWorker)
+	psReplicasCount := getReplicaCount(app.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypePS)
+	chiefCount := getReplicaCount(app.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypeChief)
+	evaluatorReplicasCount := getReplicaCount(app.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypeEval)
 
 	taskLogs, err := common.GetLogs(pluginContext, common.TensorflowTaskType, app.ObjectMeta, false,
 		*workersCount, *psReplicasCount, *chiefCount, *evaluatorReplicasCount)

flyteplugins/go/tasks/plugins/k8s/kfoperators/tensorflow/tensorflow_test.go

@@ -295,6 +295,22 @@ func dummyTensorFlowJobResource(tensorflowResourceHandler tensorflowOperatorReso
 	}
 }
 
+func TestGetReplicaCount(t *testing.T) {
+	tensorflowResourceHandler := tensorflowOperatorResourceHandler{}
+	tfObj := dummyTensorFlowCustomObj(1, 0, 0, 0)
+	taskTemplate := dummyTensorFlowTaskTemplate("the job", tfObj)
+	resource, err := tensorflowResourceHandler.BuildResource(context.TODO(), dummyTensorFlowTaskContext(taskTemplate, resourceRequirements, nil))
+	assert.NoError(t, err)
+	assert.NotNil(t, resource)
+	tensorflowJob, ok := resource.(*kubeflowv1.TFJob)
+	assert.True(t, ok)
+
+	assert.NotNil(t, getReplicaCount(tensorflowJob.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypeWorker))
+	assert.NotNil(t, getReplicaCount(tensorflowJob.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypePS))
+	assert.NotNil(t, getReplicaCount(tensorflowJob.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypeChief))
+	assert.NotNil(t, getReplicaCount(tensorflowJob.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypeEval))
+}
+
 func TestBuildResourceTensorFlow(t *testing.T) {
 	tensorflowResourceHandler := tensorflowOperatorResourceHandler{}
 

flyteplugins/go/tasks/plugins/webapi/athena/plugin_test.go

@@ -23,7 +23,6 @@ func TestCreateTaskInfo(t *testing.T) {
 	assert.Equal(t, taskInfo.ExternalResources[0].ExternalID, "query_id")
 }
 
-
 func TestCreateTaskInfoGovAWS(t *testing.T) {
 	taskInfo := createTaskInfo("query_id", awsSdk.Config{
 		Region: "us-gov-east-1",