From d335fe6ed8eb892f4713be3f00668d0c57e772ee Mon Sep 17 00:00:00 2001 From: Loc Mai Date: Tue, 27 Jun 2023 18:31:11 +0700 Subject: [PATCH] Add option for configuring HPA with custom/external metrics (#467) * feat: option for custom/external metrics HPA Signed-off-by: Loc Mai * docs: update CHANGELOG.md Signed-off-by: Loc Mai * fix: lint Signed-off-by: Loc Mai * docs: re-gen README Signed-off-by: Loc Mai * docs: update CHANGELOG Signed-off-by: Loc Mai * docs: update README format Signed-off-by: Loc Mai * fix: lint Signed-off-by: Loc Mai * chore: remove example Signed-off-by: Loc Mai * docs: shorten description Signed-off-by: Loc Mai --------- Signed-off-by: Loc Mai --- CHANGELOG.md | 1 + README.md | 6 ++++++ templates/distributor/distributor-hpa.yaml | 3 +++ templates/ingester/ingester-hpa.yaml | 3 +++ templates/nginx/nginx-hpa.yaml | 3 +++ templates/querier/querier-hpa.yaml | 3 +++ templates/ruler/ruler-hpa.yaml | 3 +++ templates/store-gateway/store-gateway-hpa.yaml | 3 +++ values.yaml | 18 ++++++++++++++++++ 9 files changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e170c1c..7b0038bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## master / unreleased +* [ENHANCEMENT] Add option to autoscale on multiple metrics and custom metrics metrics #467 * [DEPENDENCY] Update quay.io/cortexproject/cortex Docker tag to v1.15.0 #453 * [DEPENDENCY] Update quay.io/cortexproject/cortex Docker tag to v1.15.1 #456 * [DEPENDENCY] Update quay.io/cortexproject/cortex Docker tag to v1.15.2 #459 diff --git a/README.md b/README.md index e55cce6b..2ea8b083 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,7 @@ Kubernetes: `^1.19.0-0` | distributor.​annotations | object | `{}` | | | distributor.​autoscaling.​behavior | object | `{}` | Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior | | distributor.​autoscaling.​enabled | bool | `false` | Creates a HorizontalPodAutoscaler for the distributor pods. | +| distributor.​autoscaling.​extraMetrics | list | `[]` | Optional custom and external metrics for the distributor pods to scale on In order to use this option , define a list of of specific following https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics and https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects | | distributor.​autoscaling.​maxReplicas | int | `30` | | | distributor.​autoscaling.​minReplicas | int | `2` | | | distributor.​autoscaling.​targetCPUUtilizationPercentage | int | `80` | | @@ -327,6 +328,7 @@ Kubernetes: `^1.19.0-0` | ingester.​autoscaling.​behavior.​scaleDown.​stabilizationWindowSeconds | int | `3600` | uses metrics from the past 1h to make scaleDown decisions | | ingester.​autoscaling.​behavior.​scaleUp.​policies | list | `[{"periodSeconds":1800,"type":"Pods","value":1}]` | This default scaleup policy allows adding 1 pod every 30 minutes. Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior | | ingester.​autoscaling.​enabled | bool | `false` | | +| ingester.​autoscaling.​extraMetrics | list | `[]` | Optional custom and external metrics for the ingester pods to scale on In order to use this option , define a list of of specific following https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics and https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects | | ingester.​autoscaling.​maxReplicas | int | `30` | | | ingester.​autoscaling.​minReplicas | int | `3` | | | ingester.​autoscaling.​targetMemoryUtilizationPercentage | int | `80` | | @@ -425,6 +427,7 @@ Kubernetes: `^1.19.0-0` | nginx.​annotations | object | `{}` | | | nginx.​autoscaling.​behavior | object | `{}` | Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior | | nginx.​autoscaling.​enabled | bool | `false` | Creates a HorizontalPodAutoscaler for the nginx pods. | +| nginx.​autoscaling.​extraMetrics | list | `[]` | Optional custom and external metrics for the nginx pods to scale on In order to use this option , define a list of of specific following https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics and https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects | | nginx.​autoscaling.​maxReplicas | int | `30` | | | nginx.​autoscaling.​minReplicas | int | `2` | | | nginx.​autoscaling.​targetCPUUtilizationPercentage | int | `80` | | @@ -572,6 +575,7 @@ Kubernetes: `^1.19.0-0` | querier.​annotations | object | `{}` | | | querier.​autoscaling.​behavior | object | `{}` | Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior | | querier.​autoscaling.​enabled | bool | `false` | Creates a HorizontalPodAutoscaler for the querier pods. | +| querier.​autoscaling.​extraMetrics | list | `[]` | Optional custom and external metrics for the querier pods to scale on In order to use this option , define a list of of specific following https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics and https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects | | querier.​autoscaling.​maxReplicas | int | `30` | | | querier.​autoscaling.​minReplicas | int | `2` | | | querier.​autoscaling.​targetCPUUtilizationPercentage | int | `80` | | @@ -714,6 +718,7 @@ Kubernetes: `^1.19.0-0` | ruler.​annotations | object | `{}` | | | ruler.​autoscaling.​behavior | object | `{}` | Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior | | ruler.​autoscaling.​enabled | bool | `false` | Creates a HorizontalPodAutoscaler for the ruler. | +| ruler.​autoscaling.​extraMetrics | list | `[]` | Optional custom and external metrics for the ruler pods to scale on In order to use this option , define a list of of specific following https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics and https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects | | ruler.​autoscaling.​maxReplicas | int | `30` | | | ruler.​autoscaling.​minReplicas | int | `2` | | | ruler.​autoscaling.​targetCPUUtilizationPercentage | int | `80` | | @@ -792,6 +797,7 @@ Kubernetes: `^1.19.0-0` | store_gateway.​annotations | object | `{}` | | | store_gateway.​autoscaling.​behavior | object | `{}` | Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior | | store_gateway.​autoscaling.​enabled | bool | `false` | | +| store_gateway.​autoscaling.​extraMetrics | list | `[]` | Optional custom and external metrics for the store gateway pods to scale on In order to use this option , define a list of of specific following https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics and https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects | | store_gateway.​autoscaling.​maxReplicas | int | `30` | | | store_gateway.​autoscaling.​minReplicas | int | `3` | | | store_gateway.​autoscaling.​targetMemoryUtilizationPercentage | int | `80` | | diff --git a/templates/distributor/distributor-hpa.yaml b/templates/distributor/distributor-hpa.yaml index 7d089acf..02a97244 100644 --- a/templates/distributor/distributor-hpa.yaml +++ b/templates/distributor/distributor-hpa.yaml @@ -31,6 +31,9 @@ spec: type: Utilization averageUtilization: {{ . }} {{- end }} + {{- with .extraMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .behavior }} behavior: {{- toYaml . | nindent 4 }} diff --git a/templates/ingester/ingester-hpa.yaml b/templates/ingester/ingester-hpa.yaml index d4aa7b45..1bc6e6a5 100644 --- a/templates/ingester/ingester-hpa.yaml +++ b/templates/ingester/ingester-hpa.yaml @@ -21,6 +21,9 @@ spec: target: type: Utilization averageUtilization: {{ .targetMemoryUtilizationPercentage }} + {{- with .extraMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .behavior }} behavior: {{- toYaml . | nindent 4 }} diff --git a/templates/nginx/nginx-hpa.yaml b/templates/nginx/nginx-hpa.yaml index 4c632609..33a7773c 100644 --- a/templates/nginx/nginx-hpa.yaml +++ b/templates/nginx/nginx-hpa.yaml @@ -31,6 +31,9 @@ spec: type: Utilization averageUtilization: {{ . }} {{- end }} + {{- with .extraMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .behavior }} behavior: {{- toYaml . | nindent 4 }} diff --git a/templates/querier/querier-hpa.yaml b/templates/querier/querier-hpa.yaml index 88c7881f..6ff2ceb9 100644 --- a/templates/querier/querier-hpa.yaml +++ b/templates/querier/querier-hpa.yaml @@ -31,6 +31,9 @@ spec: type: Utilization averageUtilization: {{ . }} {{- end }} + {{- with .extraMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .behavior }} behavior: {{- toYaml . | nindent 4 }} diff --git a/templates/ruler/ruler-hpa.yaml b/templates/ruler/ruler-hpa.yaml index 0959c43d..909ed2eb 100644 --- a/templates/ruler/ruler-hpa.yaml +++ b/templates/ruler/ruler-hpa.yaml @@ -31,6 +31,9 @@ spec: type: Utilization averageUtilization: {{ . }} {{- end }} + {{- with .extraMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .behavior }} behavior: {{- toYaml . | nindent 4 }} diff --git a/templates/store-gateway/store-gateway-hpa.yaml b/templates/store-gateway/store-gateway-hpa.yaml index f75a21bf..b0b32892 100644 --- a/templates/store-gateway/store-gateway-hpa.yaml +++ b/templates/store-gateway/store-gateway-hpa.yaml @@ -21,6 +21,9 @@ spec: target: type: Utilization averageUtilization: {{ .targetMemoryUtilizationPercentage }} + {{- with .extraMetrics }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .behavior }} behavior: {{- toYaml . | nindent 4 }} diff --git a/values.yaml b/values.yaml index eca4b012..314bb8bf 100644 --- a/values.yaml +++ b/values.yaml @@ -372,6 +372,9 @@ distributor: targetMemoryUtilizationPercentage: 0 # 80 # -- Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior behavior: {} + # -- Optional custom and external metrics for the distributor pods to scale on + # In order to use this option , define a list of of specific following https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics and https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects + extraMetrics: [] persistentVolume: subPath: @@ -496,6 +499,9 @@ ingester: - type: Pods value: 1 periodSeconds: 1800 + # -- Optional custom and external metrics for the ingester pods to scale on + # In order to use this option , define a list of of specific following https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics and https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects + extraMetrics: [] lifecycle: # -- The /shutdown preStop hook is recommended as part of the ingester @@ -624,6 +630,9 @@ ruler: targetMemoryUtilizationPercentage: 80 # -- Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior behavior: {} + # -- Optional custom and external metrics for the ruler pods to scale on + # In order to use this option , define a list of of specific following https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics and https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects + extraMetrics: [] validation: # -- Checks that the ruler is compatible with horizontal scaling, @@ -773,6 +782,9 @@ querier: targetMemoryUtilizationPercentage: 0 # 80 # -- Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior behavior: {} + # -- Optional custom and external metrics for the querier pods to scale on + # In order to use this option , define a list of of specific following https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics and https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects + extraMetrics: [] persistentVolume: subPath: @@ -1293,6 +1305,9 @@ nginx: targetMemoryUtilizationPercentage: 0 # 80 # -- Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior behavior: {} + # -- Optional custom and external metrics for the nginx pods to scale on + # In order to use this option , define a list of of specific following https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics and https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects + extraMetrics: [] store_gateway: enabled: true @@ -1354,6 +1369,9 @@ store_gateway: targetMemoryUtilizationPercentage: 80 # -- Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior behavior: {} + # -- Optional custom and external metrics for the store gateway pods to scale on + # In order to use this option , define a list of of specific following https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics and https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects + extraMetrics: [] persistentVolume: # -- If true Store-gateway will create/use a Persistent Volume Claim