diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md index 4610da1009..418172aef0 100644 --- a/charts/flyte-core/README.md +++ b/charts/flyte-core/README.md @@ -176,6 +176,11 @@ helm install gateway bitnami/contour -n flyte | flyteadmin.serviceAccount.create | bool | `true` | Should a service account be created for flyteadmin | | flyteadmin.serviceAccount.createClusterRole | bool | `true` | Should a ClusterRole be created for Flyteadmin | | flyteadmin.serviceAccount.imagePullSecrets | list | `[]` | ImagePullSecrets to automatically assign to the service account | +| flyteadmin.serviceMonitor | object | `{"enabled":false,"interval":"60s","labels":{},"scrapeTimeout":"30s"}` | Settings for flyteadmin service monitor | +| flyteadmin.serviceMonitor.enabled | bool | `false` | If enabled create the flyteadmin service monitor | +| flyteadmin.serviceMonitor.interval | string | `"60s"` | Sets the interval at which metrics will be scraped by prometheus | +| flyteadmin.serviceMonitor.labels | object | `{}` | Sets the labels for the service monitor which are required by the prometheus to auto-detect the service monitor and start scrapping the metrics | +| flyteadmin.serviceMonitor.scrapeTimeout | string | `"30s"` | Sets the timeout after which request to scrape metrics will time out | | flyteadmin.tolerations | list | `[]` | tolerations for Flyteadmin deployment | | flyteagent.enabled | bool | `false` | | | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment | @@ -211,10 +216,17 @@ helm install gateway bitnami/contour -n flyte | flytepropeller.priorityClassName | string | `""` | Sets priorityClassName for propeller pod(s). | | flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment | | flytepropeller.resources | object | `{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"100Mi"}}` | Default resources requests and limits for Flytepropeller deployment | +| flytepropeller.service | object | `{"enabled":false}` | Settings for flytepropeller service | +| flytepropeller.service.enabled | bool | `false` | If enabled create the flytepropeller service | | flytepropeller.serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":[]}` | Configuration for service accounts for FlytePropeller | | flytepropeller.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to FlytePropeller pods | | flytepropeller.serviceAccount.create | bool | `true` | Should a service account be created for FlytePropeller | | flytepropeller.serviceAccount.imagePullSecrets | list | `[]` | ImagePullSecrets to automatically assign to the service account | +| flytepropeller.serviceMonitor | object | `{"enabled":false,"interval":"60s","labels":{},"scrapeTimeout":"30s"}` | Settings for flytepropeller service monitor | +| flytepropeller.serviceMonitor.enabled | bool | `false` | If enabled create the flyetepropeller service monitor | +| flytepropeller.serviceMonitor.interval | string | `"60s"` | Sets the interval at which metrics will be scraped by prometheus | +| flytepropeller.serviceMonitor.labels | object | `{}` | Sets the labels for the service monitor which are required by the prometheus to auto-detect the service monitor and start scrapping the metrics | +| flytepropeller.serviceMonitor.scrapeTimeout | string | `"30s"` | Sets the timeout after which request to scrape metrics will time out | | flytepropeller.terminationMessagePolicy | string | `"FallbackToLogsOnError"` | Error reporting | | flytepropeller.tolerations | list | `[]` | tolerations for Flytepropeller deployment | | flytescheduler.additionalContainers | list | `[]` | Appends additional containers to the deployment spec. May include template values. | diff --git a/charts/flyte-core/templates/admin/service-monitor.yaml b/charts/flyte-core/templates/admin/service-monitor.yaml new file mode 100644 index 0000000000..7d5be0352f --- /dev/null +++ b/charts/flyte-core/templates/admin/service-monitor.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.flyteadmin.serviceMonitor.enabled .Values.flyteadmin.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "flyteadmin.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: + {{- with .Values.flyteadmin.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - interval: {{ .Values.flyteadmin.serviceMonitor.interval }} + port: http-metrics + path: /metrics + scrapeTimeout: {{ .Values.flyteadmin.serviceMonitor.scrapeTimeout }} + selector: + matchLabels: {{ include "flyteadmin.selectorLabels" . | nindent 6 }} +{{- end }} \ No newline at end of file diff --git a/charts/flyte-core/templates/propeller/service-monitor.yaml b/charts/flyte-core/templates/propeller/service-monitor.yaml new file mode 100644 index 0000000000..9a1e5c2eca --- /dev/null +++ b/charts/flyte-core/templates/propeller/service-monitor.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.flytepropeller.service.enabled .Values.flytepropeller.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + namespace: {{ template "flyte.namespace" . }} + name: {{ template "flytepropeller.name" . }} + labels: + {{- with .Values.flytepropeller.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - interval: {{ .Values.flytepropeller.serviceMonitor.interval }} + port: http-metrics + path: /metrics + scrapeTimeout: {{ .Values.flytepropeller.serviceMonitor.scrapeTimeout }} + selector: + matchLabels: {{ include "flytepropeller.selectorLabels" . | nindent 6 }} +{{- end }} \ No newline at end of file diff --git a/charts/flyte-core/templates/propeller/service.yaml b/charts/flyte-core/templates/propeller/service.yaml new file mode 100644 index 0000000000..7569f3e6fc --- /dev/null +++ b/charts/flyte-core/templates/propeller/service.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.flytepropeller.enabled .Values.flytepropeller.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + namespace: {{ template "flyte.namespace" . }} + name: {{ template "flytepropeller.name" . }} + labels: {{ include "flytepropeller.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http-metrics + protocol: TCP + port: 10254 + selector: {{ include "flytepropeller.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/flyte-core/values-eks.yaml b/charts/flyte-core/values-eks.yaml index 7b7ca446f5..03276598bf 100644 --- a/charts/flyte-core/values-eks.yaml +++ b/charts/flyte-core/values-eks.yaml @@ -23,6 +23,9 @@ flyteadmin: annotations: eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.userSettings.accountNumber }}:role/iam-role-flyte + serviceMonitor: + enabled: false + resources: limits: ephemeral-storage: 200Mi @@ -78,6 +81,10 @@ datacatalog: flytepropeller: replicaCount: 2 manager: false + serviceMonitor: + enabled: false + service: + enabled: false serviceAccount: # -- If the service account is created by you, make this false create: true diff --git a/charts/flyte-core/values-gcp.yaml b/charts/flyte-core/values-gcp.yaml index 1a53510062..4051fb23b9 100644 --- a/charts/flyte-core/values-gcp.yaml +++ b/charts/flyte-core/values-gcp.yaml @@ -14,6 +14,8 @@ userSettings: flyteadmin: replicaCount: 1 + serviceMonitor: + enabled: false serviceAccount: # -- If the service account is created by you, make this false, else a new service account will be created and the flyteadmin role will be added # you can change the name of this role @@ -82,6 +84,10 @@ datacatalog: flytepropeller: replicaCount: 1 manager: false + serviceMonitor: + enabled: false + service: + enabled: false serviceAccount: # -- If the service account is created by you, make this false, else a new service account will be created and the iam-role-flyte will be added # you can change the name of this role diff --git a/charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml b/charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml index c3ed3f5ddd..380db08be7 100644 --- a/charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml +++ b/charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml @@ -11,6 +11,8 @@ flyteadmin: enabled: true + serviceMonitor: + enabled: false # -- Replicas count for Flyteadmin deployment replicaCount: 1 image: diff --git a/charts/flyte-core/values-sandbox.yaml b/charts/flyte-core/values-sandbox.yaml index 31ee4cedc6..c0e6d38404 100644 --- a/charts/flyte-core/values-sandbox.yaml +++ b/charts/flyte-core/values-sandbox.yaml @@ -1,5 +1,7 @@ flyteadmin: + serviceMonitor: + enabled: false service: annotations: projectcontour.io/upstream-protocol.h2c: grpc diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml index 7ad1707e89..6b5d7ea58e 100755 --- a/charts/flyte-core/values.yaml +++ b/charts/flyte-core/values.yaml @@ -110,6 +110,17 @@ flyteadmin: # -- Sets priorityClassName for flyteadmin pod(s). priorityClassName: "" + # -- Settings for flyteadmin service monitor + serviceMonitor: + # -- If enabled create the flyteadmin service monitor + enabled: false + # -- Sets the interval at which metrics will be scraped by prometheus + interval: 60s + # -- Sets the timeout after which request to scrape metrics will time out + scrapeTimeout: 30s + # -- Sets the labels for the service monitor which are required by the + # prometheus to auto-detect the service monitor and start scrapping the metrics + labels: {} # # FLYTESCHEDULER SETTINGS # @@ -290,6 +301,22 @@ flytepropeller: # -- Sets priorityClassName for propeller pod(s). priorityClassName: "" + # -- Settings for flytepropeller service + service: + # -- If enabled create the flytepropeller service + enabled: false + + # -- Settings for flytepropeller service monitor + serviceMonitor: + # -- If enabled create the flyetepropeller service monitor + enabled: false + # -- Sets the labels for the service monitor which are required by the + # prometheus to auto-detect the service monitor and start scrapping the metrics + labels: {} + # -- Sets the interval at which metrics will be scraped by prometheus + interval: 60s + # -- Sets the timeout after which request to scrape metrics will time out + scrapeTimeout: 30s # # FLYTECONSOLE SETTINGS # @@ -895,4 +922,4 @@ databricks: databricks: entrypointFile: dbfs:///FileStore/tables/entrypoint.py # Databricks account - databricksInstance: dbc-a53b7a3c-614c + databricksInstance: dbc-a53b7a3c-614c \ No newline at end of file diff --git a/flyteplugins/go/tasks/plugins/k8s/kfoperators/tensorflow/tensorflow.go b/flyteplugins/go/tasks/plugins/k8s/kfoperators/tensorflow/tensorflow.go index 8db340d37e..d3e05ac0d6 100644 --- a/flyteplugins/go/tasks/plugins/k8s/kfoperators/tensorflow/tensorflow.go +++ b/flyteplugins/go/tasks/plugins/k8s/kfoperators/tensorflow/tensorflow.go @@ -217,16 +217,24 @@ func (tensorflowOperatorResourceHandler) BuildResource(ctx context.Context, task return job, nil } +func getReplicaCount(specs map[commonOp.ReplicaType]*commonOp.ReplicaSpec, replicaType commonOp.ReplicaType) *int32 { + if spec, ok := specs[replicaType]; ok && spec.Replicas != nil { + return spec.Replicas + } + + return new(int32) // return 0 as default value +} + // Analyses the k8s resource and reports the status as TaskPhase. This call is expected to be relatively fast, // any operations that might take a long time (limits are configured system-wide) should be offloaded to the // background. func (tensorflowOperatorResourceHandler) GetTaskPhase(_ context.Context, pluginContext k8s.PluginContext, resource client.Object) (pluginsCore.PhaseInfo, error) { app := resource.(*kubeflowv1.TFJob) - workersCount := app.Spec.TFReplicaSpecs[kubeflowv1.TFJobReplicaTypeWorker].Replicas - psReplicasCount := app.Spec.TFReplicaSpecs[kubeflowv1.TFJobReplicaTypePS].Replicas - chiefCount := app.Spec.TFReplicaSpecs[kubeflowv1.TFJobReplicaTypeChief].Replicas - evaluatorReplicasCount := app.Spec.TFReplicaSpecs[kubeflowv1.TFJobReplicaTypeEval].Replicas + workersCount := getReplicaCount(app.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypeWorker) + psReplicasCount := getReplicaCount(app.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypePS) + chiefCount := getReplicaCount(app.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypeChief) + evaluatorReplicasCount := getReplicaCount(app.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypeEval) taskLogs, err := common.GetLogs(pluginContext, common.TensorflowTaskType, app.ObjectMeta, false, *workersCount, *psReplicasCount, *chiefCount, *evaluatorReplicasCount) diff --git a/flyteplugins/go/tasks/plugins/k8s/kfoperators/tensorflow/tensorflow_test.go b/flyteplugins/go/tasks/plugins/k8s/kfoperators/tensorflow/tensorflow_test.go index bcabdaa87f..6867411f06 100644 --- a/flyteplugins/go/tasks/plugins/k8s/kfoperators/tensorflow/tensorflow_test.go +++ b/flyteplugins/go/tasks/plugins/k8s/kfoperators/tensorflow/tensorflow_test.go @@ -295,6 +295,22 @@ func dummyTensorFlowJobResource(tensorflowResourceHandler tensorflowOperatorReso } } +func TestGetReplicaCount(t *testing.T) { + tensorflowResourceHandler := tensorflowOperatorResourceHandler{} + tfObj := dummyTensorFlowCustomObj(1, 0, 0, 0) + taskTemplate := dummyTensorFlowTaskTemplate("the job", tfObj) + resource, err := tensorflowResourceHandler.BuildResource(context.TODO(), dummyTensorFlowTaskContext(taskTemplate, resourceRequirements, nil)) + assert.NoError(t, err) + assert.NotNil(t, resource) + tensorflowJob, ok := resource.(*kubeflowv1.TFJob) + assert.True(t, ok) + + assert.NotNil(t, getReplicaCount(tensorflowJob.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypeWorker)) + assert.NotNil(t, getReplicaCount(tensorflowJob.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypePS)) + assert.NotNil(t, getReplicaCount(tensorflowJob.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypeChief)) + assert.NotNil(t, getReplicaCount(tensorflowJob.Spec.TFReplicaSpecs, kubeflowv1.TFJobReplicaTypeEval)) +} + func TestBuildResourceTensorFlow(t *testing.T) { tensorflowResourceHandler := tensorflowOperatorResourceHandler{} diff --git a/flyteplugins/go/tasks/plugins/webapi/athena/plugin_test.go b/flyteplugins/go/tasks/plugins/webapi/athena/plugin_test.go index e19829447e..c3fc39f451 100644 --- a/flyteplugins/go/tasks/plugins/webapi/athena/plugin_test.go +++ b/flyteplugins/go/tasks/plugins/webapi/athena/plugin_test.go @@ -23,7 +23,6 @@ func TestCreateTaskInfo(t *testing.T) { assert.Equal(t, taskInfo.ExternalResources[0].ExternalID, "query_id") } - func TestCreateTaskInfoGovAWS(t *testing.T) { taskInfo := createTaskInfo("query_id", awsSdk.Config{ Region: "us-gov-east-1",