diff --git a/apis/v1alpha2/nginxproxy_types.go b/apis/v1alpha2/nginxproxy_types.go index 43b509d06d..48616f8545 100644 --- a/apis/v1alpha2/nginxproxy_types.go +++ b/apis/v1alpha2/nginxproxy_types.go @@ -1,6 +1,7 @@ package v1alpha2 import ( + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -388,6 +389,11 @@ type DeploymentSpec struct { // +optional Replicas *int32 `json:"replicas,omitempty"` + // Autoscaling defines the configuration for Horizontal Pod Autoscaling. + // + // +optional + Autoscaling HPASpec `json:"autoscaling"` + // Pod defines Pod-specific fields. // // +optional @@ -412,6 +418,56 @@ type DaemonSetSpec struct { Container ContainerSpec `json:"container"` } +// +kubebuilder:validation:XValidation:message="at least one metric must be specified when autoscaling is enabled",rule="!self.enabled || (has(self.targetCPUUtilizationPercentage) || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) && size(self.autoscalingTemplate) > 0))" +// +kubebuilder:validation:XValidation:message="minReplicas must be less than or equal to maxReplicas",rule="self.minReplicas <= self.maxReplicas" +// +kubebuilder:validation:XValidation:message="CPU utilization must be between 1 and 100",rule="!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage >= 1 && self.targetCPUUtilizationPercentage <= 100)" +// +kubebuilder:validation:XValidation:message="memory utilization must be between 1 and 100",rule="!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage >= 1 && self.targetMemoryUtilizationPercentage <= 100)" +// +// HPASpec is the configuration for the Horizontal Pod Autoscaling. +// +//nolint:lll +type HPASpec struct { + // Behavior configures the scaling behavior of the target + // in both Up and Down directions (scaleUp and scaleDown fields respectively). + // If not set, the default HPAScalingRules for scale up and scale down are used. + // + // +optional + Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"` + + // AutoscalingTemplate configures the additional scaling option. + // + // +optional + AutoscalingTemplate *[]autoscalingv2.MetricSpec `json:"autoscalingTemplate,omitempty"` + + // Target cpu utilization percentage of HPA. + // + // +optional + TargetCPUUtilizationPercentage *int32 `json:"targetCPUUtilizationPercentage,omitempty"` + + // Target memory utilization percentage of HPA. + // + // +optional + TargetMemoryUtilizationPercentage *int32 `json:"targetMemoryUtilizationPercentage,omitempty"` + + // Annotation for Horizontal Pod Autoscaler + // Annotations is an unstructured key value map stored with a resource that may be + // set by external tools to store and retrieve arbitrary metadata. They are not + // queryable and should be preserved when modifying objects. + // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + // + // +optional + HPAAnnotations map[string]string `json:"hpaAnnotations,omitempty"` + + // Minimum number of replicas. + MinReplicas int32 `json:"minReplicas"` + + // Maximum number of replicas. + MaxReplicas int32 `json:"maxReplicas"` + + // Enable or disable Horizontal Pod Autoscaler + Enabled bool `json:"enabled"` +} + // PodSpec defines Pod-specific fields. type PodSpec struct { // TerminationGracePeriodSeconds is the optional duration in seconds the pod needs to terminate gracefully. diff --git a/apis/v1alpha2/zz_generated.deepcopy.go b/apis/v1alpha2/zz_generated.deepcopy.go index 4b0f8bf9f8..e4e4981ca4 100644 --- a/apis/v1alpha2/zz_generated.deepcopy.go +++ b/apis/v1alpha2/zz_generated.deepcopy.go @@ -6,6 +6,7 @@ package v1alpha2 import ( "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1" + "k8s.io/api/autoscaling/v2" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" apisv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -78,6 +79,7 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { *out = new(int32) **out = **in } + in.Autoscaling.DeepCopyInto(&out.Autoscaling) in.Pod.DeepCopyInto(&out.Pod) in.Container.DeepCopyInto(&out.Container) } @@ -92,6 +94,54 @@ func (in *DeploymentSpec) DeepCopy() *DeploymentSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HPASpec) DeepCopyInto(out *HPASpec) { + *out = *in + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(v2.HorizontalPodAutoscalerBehavior) + (*in).DeepCopyInto(*out) + } + if in.AutoscalingTemplate != nil { + in, out := &in.AutoscalingTemplate, &out.AutoscalingTemplate + *out = new([]v2.MetricSpec) + if **in != nil { + in, out := *in, *out + *out = make([]v2.MetricSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + } + if in.TargetCPUUtilizationPercentage != nil { + in, out := &in.TargetCPUUtilizationPercentage, &out.TargetCPUUtilizationPercentage + *out = new(int32) + **out = **in + } + if in.TargetMemoryUtilizationPercentage != nil { + in, out := &in.TargetMemoryUtilizationPercentage, &out.TargetMemoryUtilizationPercentage + *out = new(int32) + **out = **in + } + if in.HPAAnnotations != nil { + in, out := &in.HPAAnnotations, &out.HPAAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HPASpec. +func (in *HPASpec) DeepCopy() *HPASpec { + if in == nil { + return nil + } + out := new(HPASpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Image) DeepCopyInto(out *Image) { *out = *in diff --git a/charts/nginx-gateway-fabric/README.md b/charts/nginx-gateway-fabric/README.md index 3fb3f6230b..7c4c348c6a 100644 --- a/charts/nginx-gateway-fabric/README.md +++ b/charts/nginx-gateway-fabric/README.md @@ -264,7 +264,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `certGenerator.ttlSecondsAfterFinished` | How long to wait after the cert generator job has finished before it is removed by the job controller. | int | `30` | | `clusterDomain` | The DNS cluster domain of your Kubernetes cluster. | string | `"cluster.local"` | | `gateways` | A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. | list | `[]` | -| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` | +| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"autoscaling":{"enabled":false},"autoscalingTemplate":[],"config":{},"container":{},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` | | `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` | | `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` | | `nginx.debug` | Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. | bool | `false` | @@ -288,7 +288,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `nginx.usage.resolver` | The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. | string | `""` | | `nginx.usage.secretName` | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"nplus-license"` | | `nginx.usage.skipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` | -| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` | +| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"autoscaling":{"enabled":false},"autoscalingTemplate":[],"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` | | `nginxGateway.affinity` | The affinity of the NGINX Gateway Fabric control plane pod. | object | `{}` | | `nginxGateway.config.logging.level` | Log level. | string | `"info"` | | `nginxGateway.configAnnotations` | Set of custom annotations for NginxGateway objects. | object | `{}` | diff --git a/charts/nginx-gateway-fabric/templates/clusterrole.yaml b/charts/nginx-gateway-fabric/templates/clusterrole.yaml index 1205570535..84d32a1fc8 100644 --- a/charts/nginx-gateway-fabric/templates/clusterrole.yaml +++ b/charts/nginx-gateway-fabric/templates/clusterrole.yaml @@ -8,6 +8,9 @@ rules: - apiGroups: - "" - apps + {{- if or .Values.nginx.autoscaling.enabled .Values.nginxGateway.autoscaling.enabled }} + - autoscaling + {{- end }} resources: - secrets - configmaps @@ -15,6 +18,9 @@ rules: - services - deployments - daemonsets + {{- if or .Values.nginx.autoscaling.enabled .Values.nginxGateway.autoscaling.enabled }} + - horizontalpodautoscalers + {{- end }} verbs: - create - update diff --git a/charts/nginx-gateway-fabric/templates/hpa.yaml b/charts/nginx-gateway-fabric/templates/hpa.yaml new file mode 100644 index 0000000000..86730bcfa5 --- /dev/null +++ b/charts/nginx-gateway-fabric/templates/hpa.yaml @@ -0,0 +1,46 @@ +{{- if and (eq .Values.nginxGateway.kind "deployment") .Values.nginxGateway.autoscaling.enabled (.Capabilities.APIVersions.Has "autoscaling/v2") -}} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + {{- with .Values.nginxGateway.autoscaling.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "nginx-gateway.labels" . | nindent 4 }} + {{- with .Values.nginxGateway.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "nginx-gateway.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "nginx-gateway.fullname" . }} + minReplicas: {{ .Values.nginxGateway.autoscaling.minReplicas }} + maxReplicas: {{ .Values.nginxGateway.autoscaling.maxReplicas }} + metrics: + {{- with .Values.nginxGateway.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.nginxGateway.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.autoscalingTemplate }} + {{- toYaml . | nindent 2 }} + {{- end }} + {{- with .Values.nginxGateway.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml index b5e33292c8..29ab315c76 100644 --- a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml +++ b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml @@ -12,7 +12,33 @@ spec: kubernetes: {{- if eq .Values.nginx.kind "deployment" }} deployment: + {{- if .Values.nginx.replicas }} replicas: {{ .Values.nginx.replicas }} + {{- end }} + {{- if .Values.nginx.autoscaling.enabled }} + autoscaling: + enabled: {{ .Values.nginx.autoscaling.enabled }} + {{- if .Values.nginx.autoscaling.hpaAnnotations }} + hpaAnnotations: + {{- toYaml .Values.nginx.autoscaling.hpaAnnotations | nindent 10 }} + {{- end }} + minReplicas: {{ .Values.nginx.autoscaling.minReplicas }} + maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }} + {{- if .Values.nginx.autoscaling.targetCPUUtilizationPercentage }} + targetCPUUtilizationPercentage: {{ .Values.nginx.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }} + targetMemoryUtilizationPercentage: {{ .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} + {{- if .Values.nginx.autoscaling.behavior }} + behavior: + {{- toYaml .Values.nginx.autoscaling.behavior | nindent 10 }} + {{- end }} + {{- if .Values.nginx.autoscalingTemplate }} + autoscalingTemplate: + {{- toYaml .Values.nginx.autoscalingTemplate | nindent 8 }} + {{- end }} + {{- end }} {{- if .Values.nginx.pod }} pod: {{- toYaml .Values.nginx.pod | nindent 8 }} diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index 15582052e5..bd86e4d43e 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -98,6 +98,28 @@ "nginx": { "description": "The nginx section contains the configuration for all NGINX data plane deployments\ninstalled by the NGINX Gateway Fabric control plane.", "properties": { + "autoscaling": { + "properties": { + "enabled": { + "default": false, + "description": "Enable or disable Horizontal Pod Autoscaler", + "required": [], + "title": "enabled", + "type": "boolean" + } + }, + "required": [], + "title": "autoscaling", + "type": "object" + }, + "autoscalingTemplate": { + "items": { + "required": [] + }, + "required": [], + "title": "autoscalingTemplate", + "type": "array" + }, "config": { "description": "The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways\nmanaged by this instance of NGINX Gateway Fabric.", "properties": { @@ -325,6 +347,7 @@ "type": "boolean" }, "image": { + "description": "Custom or additional autoscaling metrics\nref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics\n- type: Pods\n pods:\n metric:\n name: nginx_gateway_fabric_nginx_process_requests_total\n target:\n type: AverageValue\n averageValue: 10000m", "properties": { "pullPolicy": { "default": "Always", @@ -540,6 +563,28 @@ "title": "affinity", "type": "object" }, + "autoscaling": { + "properties": { + "enabled": { + "default": false, + "description": "Enable or disable Horizontal Pod Autoscaler", + "required": [], + "title": "enabled", + "type": "boolean" + } + }, + "required": [], + "title": "autoscaling", + "type": "object" + }, + "autoscalingTemplate": { + "items": { + "required": [] + }, + "required": [], + "title": "autoscalingTemplate", + "type": "array" + }, "config": { "description": "The dynamic configuration for the control plane that is contained in the NginxGateway resource.", "properties": { diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index 82c427fd6b..88515bc570 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -159,6 +159,38 @@ nginxGateway: # -- The topology spread constraints for the NGINX Gateway Fabric control plane pod. topologySpreadConstraints: [] + autoscaling: + # Enable or disable Horizontal Pod Autoscaler + enabled: false + # annotations: {} + # minReplicas: 1 + # maxReplicas: 11 + # targetCPUUtilizationPercentage: 50 + # targetMemoryUtilizationPercentage: 50 + # behavior: {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + autoscalingTemplate: [] + # Custom or additional autoscaling metrics + # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics + # - type: Pods + # pods: + # metric: + # name: nginx_gateway_fabric_nginx_process_requests_total + # target: + # type: AverageValue + # averageValue: 10000m + metrics: # -- Enable exposing metrics in the Prometheus format. enable: true @@ -199,6 +231,37 @@ nginx: # -- The number of replicas of the NGINX Deployment. replicas: 1 + autoscaling: + # Enable or disable Horizontal Pod Autoscaler + enabled: false + # hpaAnnotations: {} + # minReplicas: 1 + # maxReplicas: 11 + # targetCPUUtilizationPercentage: 50 + # targetMemoryUtilizationPercentage: 50 + # behavior: {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + autoscalingTemplate: [] + # Custom or additional autoscaling metrics + # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics + # - type: Pods + # pods: + # metric: + # name: nginx_gateway_fabric_nginx_process_requests_total + # target: + # type: AverageValue + # averageValue: 10000m image: # -- The NGINX image to use. repository: ghcr.io/nginx/nginx-gateway-fabric/nginx @@ -403,7 +466,7 @@ nginx: # -- The container configuration for the NGINX container. This is applied globally to all Gateways managed by this # instance of NGINX Gateway Fabric. container: {} - # -- The resource requirements of the NGINX container. + # -- The resource requirements of the NGINX container. You should set this value if you want to use dataplane Autoscaling(HPA). # resources: {} # -- The lifecycle of the NGINX container. diff --git a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml index 9947e34eb7..4ba097de57 100644 --- a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml +++ b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml @@ -3459,6 +3459,693 @@ spec: Deployment is the configuration for the NGINX Deployment. This is the default deployment option. properties: + autoscaling: + description: Autoscaling defines the configuration for Horizontal + Pod Autoscaling. + properties: + autoscalingTemplate: + description: AutoscalingTemplate configures the additional + scaling option. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + type: string + required: + - type + type: object + type: array + behavior: + description: |- + Behavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + If not set, the default HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + enabled: + description: Enable or disable Horizontal Pod Autoscaler + type: boolean + hpaAnnotations: + additionalProperties: + type: string + description: |- + Annotation for Horizontal Pod Autoscaler + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + maxReplicas: + description: Maximum number of replicas. + format: int32 + type: integer + minReplicas: + description: Minimum number of replicas. + format: int32 + type: integer + targetCPUUtilizationPercentage: + description: Target cpu utilization percentage of HPA. + format: int32 + type: integer + targetMemoryUtilizationPercentage: + description: Target memory utilization percentage of HPA. + format: int32 + type: integer + required: + - enabled + - maxReplicas + - minReplicas + type: object + x-kubernetes-validations: + - message: at least one metric must be specified when autoscaling + is enabled + rule: '!self.enabled || (has(self.targetCPUUtilizationPercentage) + || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) + && size(self.autoscalingTemplate) > 0))' + - message: minReplicas must be less than or equal to maxReplicas + rule: self.minReplicas <= self.maxReplicas + - message: CPU utilization must be between 1 and 100 + rule: '!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage + >= 1 && self.targetCPUUtilizationPercentage <= 100)' + - message: memory utilization must be between 1 and 100 + rule: '!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage + >= 1 && self.targetMemoryUtilizationPercentage <= 100)' container: description: Container defines container fields for the NGINX container. diff --git a/deploy/crds.yaml b/deploy/crds.yaml index 7517ce1c4a..2b9568aa83 100644 --- a/deploy/crds.yaml +++ b/deploy/crds.yaml @@ -4044,6 +4044,693 @@ spec: Deployment is the configuration for the NGINX Deployment. This is the default deployment option. properties: + autoscaling: + description: Autoscaling defines the configuration for Horizontal + Pod Autoscaling. + properties: + autoscalingTemplate: + description: AutoscalingTemplate configures the additional + scaling option. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + type: string + required: + - type + type: object + type: array + behavior: + description: |- + Behavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + If not set, the default HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + enabled: + description: Enable or disable Horizontal Pod Autoscaler + type: boolean + hpaAnnotations: + additionalProperties: + type: string + description: |- + Annotation for Horizontal Pod Autoscaler + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + maxReplicas: + description: Maximum number of replicas. + format: int32 + type: integer + minReplicas: + description: Minimum number of replicas. + format: int32 + type: integer + targetCPUUtilizationPercentage: + description: Target cpu utilization percentage of HPA. + format: int32 + type: integer + targetMemoryUtilizationPercentage: + description: Target memory utilization percentage of HPA. + format: int32 + type: integer + required: + - enabled + - maxReplicas + - minReplicas + type: object + x-kubernetes-validations: + - message: at least one metric must be specified when autoscaling + is enabled + rule: '!self.enabled || (has(self.targetCPUUtilizationPercentage) + || has(self.targetMemoryUtilizationPercentage) || (has(self.autoscalingTemplate) + && size(self.autoscalingTemplate) > 0))' + - message: minReplicas must be less than or equal to maxReplicas + rule: self.minReplicas <= self.maxReplicas + - message: CPU utilization must be between 1 and 100 + rule: '!has(self.targetCPUUtilizationPercentage) || (self.targetCPUUtilizationPercentage + >= 1 && self.targetCPUUtilizationPercentage <= 100)' + - message: memory utilization must be between 1 and 100 + rule: '!has(self.targetMemoryUtilizationPercentage) || (self.targetMemoryUtilizationPercentage + >= 1 && self.targetMemoryUtilizationPercentage <= 100)' container: description: Container defines container fields for the NGINX container. diff --git a/internal/controller/manager.go b/internal/controller/manager.go index f7bc0a68e5..bebc0a1eec 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -12,6 +12,7 @@ import ( "google.golang.org/grpc" appsv1 "k8s.io/api/apps/v1" authv1 "k8s.io/api/authentication/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" apiv1 "k8s.io/api/core/v1" discoveryV1 "k8s.io/api/discovery/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -91,6 +92,7 @@ func init() { utilruntime.Must(ngfAPIv1alpha2.AddToScheme(scheme)) utilruntime.Must(apiext.AddToScheme(scheme)) utilruntime.Must(appsv1.AddToScheme(scheme)) + utilruntime.Must(autoscalingv2.AddToScheme(scheme)) utilruntime.Must(authv1.AddToScheme(scheme)) utilruntime.Must(rbacv1.AddToScheme(scheme)) } diff --git a/internal/controller/provisioner/eventloop.go b/internal/controller/provisioner/eventloop.go index 7b85a47780..6f38d75c7c 100644 --- a/internal/controller/provisioner/eventloop.go +++ b/internal/controller/provisioner/eventloop.go @@ -6,6 +6,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,6 +74,18 @@ func newEventLoop( ), }, }, + { + objectType: &autoscalingv2.HorizontalPodAutoscaler{}, + options: []controller.Option{ + controller.WithK8sPredicate( + k8spredicate.And( + k8spredicate.GenerationChangedPredicate{}, + nginxResourceLabelPredicate, + predicate.RestartDeploymentAnnotationPredicate{}, + ), + ), + }, + }, { objectType: &appsv1.DaemonSet{}, options: []controller.Option{ @@ -184,6 +197,7 @@ func newEventLoop( // to provision or deprovision any nginx resources. &gatewayv1.GatewayList{}, &appsv1.DeploymentList{}, + &autoscalingv2.HorizontalPodAutoscalerList{}, &corev1.ServiceList{}, &corev1.ServiceAccountList{}, &corev1.ConfigMapList{}, diff --git a/internal/controller/provisioner/handler.go b/internal/controller/provisioner/handler.go index c5f3f2f4a9..f84160ca09 100644 --- a/internal/controller/provisioner/handler.go +++ b/internal/controller/provisioner/handler.go @@ -9,6 +9,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -62,7 +63,7 @@ func (h *eventHandler) HandleEventBatch(ctx context.Context, logger logr.Logger, case *gatewayv1.Gateway: h.store.updateGateway(obj) case *appsv1.Deployment, *appsv1.DaemonSet, *corev1.ServiceAccount, - *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding: + *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding, *autoscalingv2.HorizontalPodAutoscaler: objLabels := labels.Set(obj.GetLabels()) if h.labelSelector.Matches(objLabels) { gatewayName := objLabels.Get(controller.GatewayLabel) @@ -118,7 +119,7 @@ func (h *eventHandler) HandleEventBatch(ctx context.Context, logger logr.Logger, } h.store.deleteGateway(e.NamespacedName) case *appsv1.Deployment, *appsv1.DaemonSet, *corev1.Service, *corev1.ServiceAccount, - *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding: + *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding, *autoscalingv2.HorizontalPodAutoscaler: if err := h.reprovisionResources(ctx, e); err != nil { logger.Error(err, "error re-provisioning nginx resources") } diff --git a/internal/controller/provisioner/objects.go b/internal/controller/provisioner/objects.go index 3b43ac8b66..5676144d15 100644 --- a/internal/controller/provisioner/objects.go +++ b/internal/controller/provisioner/objects.go @@ -10,6 +10,7 @@ import ( "time" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -152,6 +153,7 @@ func (p *NginxProvisioner) buildNginxResourceObjects( // role/binding (if openshift) // service // deployment/daemonset + // hpa objects := make([]client.Object, 0, len(configmaps)+len(secrets)+len(openshiftObjs)+3) objects = append(objects, secrets...) @@ -160,11 +162,31 @@ func (p *NginxProvisioner) buildNginxResourceObjects( if p.isOpenshift { objects = append(objects, openshiftObjs...) } + objects = append(objects, service, deployment) + if hpa := p.buildHPA(objectMeta, nProxyCfg); hpa != nil { + objects = append(objects, hpa) + } + return objects, err } +func isAutoscalingEnabled(dep *ngfAPIv1alpha2.DeploymentSpec) bool { + return dep != nil && dep.Autoscaling.Enabled +} + +func (p *NginxProvisioner) buildHPA( + objectMeta metav1.ObjectMeta, + nProxyCfg *graph.EffectiveNginxProxy, +) client.Object { + if nProxyCfg == nil || nProxyCfg.Kubernetes == nil || !isAutoscalingEnabled(nProxyCfg.Kubernetes.Deployment) { + return nil + } + + return buildNginxDeploymentHPA(objectMeta, nProxyCfg.Kubernetes.Deployment.Autoscaling) +} + func (p *NginxProvisioner) buildNginxSecrets( objectMeta metav1.ObjectMeta, agentTLSSecretName string, @@ -895,6 +917,154 @@ func (p *NginxProvisioner) buildImage(nProxyCfg *graph.EffectiveNginxProxy) (str return fmt.Sprintf("%s:%s", image, tag), pullPolicy } +func getMetricTargetByType(target autoscalingv2.MetricTarget) autoscalingv2.MetricTarget { + switch target.Type { + case autoscalingv2.UtilizationMetricType: + return autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: target.AverageUtilization, + } + case autoscalingv2.AverageValueMetricType: + return autoscalingv2.MetricTarget{ + Type: autoscalingv2.AverageValueMetricType, + AverageValue: target.AverageValue, + } + case autoscalingv2.ValueMetricType: + return autoscalingv2.MetricTarget{ + Type: autoscalingv2.ValueMetricType, + Value: target.Value, + } + default: + return autoscalingv2.MetricTarget{} + } +} + +func buildNginxDeploymentHPA( + objectMeta metav1.ObjectMeta, + autoScaling ngfAPIv1alpha2.HPASpec, +) *autoscalingv2.HorizontalPodAutoscaler { + objectMeta.Annotations = autoScaling.HPAAnnotations + + if !autoScaling.Enabled { + return nil + } + var metrics []autoscalingv2.MetricSpec + + cpuUtil := autoScaling.TargetCPUUtilizationPercentage + memUtil := autoScaling.TargetMemoryUtilizationPercentage + autoscalingTemplate := autoScaling.AutoscalingTemplate + + if cpuUtil != nil { + metrics = append(metrics, autoscalingv2.MetricSpec{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: "cpu", + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: cpuUtil, + }, + }, + }) + } + + if memUtil != nil { + metrics = append(metrics, autoscalingv2.MetricSpec{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: "memory", + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: memUtil, + }, + }, + }) + } + + if autoscalingTemplate != nil { + for _, additionalAutoscaling := range *autoscalingTemplate { + metric := buildAdditionalMetric(additionalAutoscaling) + if metric != nil { + metrics = append(metrics, *metric) + } + } + } + + if len(metrics) == 0 { + // No metrics configured, skip HPA creation + return nil + } + + return &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: objectMeta, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: objectMeta.Name, + }, + MinReplicas: &autoScaling.MinReplicas, + MaxReplicas: autoScaling.MaxReplicas, + Metrics: metrics, + Behavior: autoScaling.Behavior, + }, + } +} + +func buildAdditionalMetric(additionalAutoscaling autoscalingv2.MetricSpec) *autoscalingv2.MetricSpec { + switch additionalAutoscaling.Type { + case autoscalingv2.ResourceMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: additionalAutoscaling.Resource.Name, + Target: getMetricTargetByType(additionalAutoscaling.Resource.Target), + }, + } + case autoscalingv2.PodsMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + Pods: &autoscalingv2.PodsMetricSource{ + Metric: additionalAutoscaling.Pods.Metric, + Target: getMetricTargetByType(additionalAutoscaling.Pods.Target), + }, + } + case autoscalingv2.ContainerResourceMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + ContainerResource: &autoscalingv2.ContainerResourceMetricSource{ + Name: additionalAutoscaling.ContainerResource.Name, + Target: getMetricTargetByType(additionalAutoscaling.ContainerResource.Target), + Container: additionalAutoscaling.ContainerResource.Container, + }, + } + case autoscalingv2.ObjectMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + Object: &autoscalingv2.ObjectMetricSource{ + DescribedObject: additionalAutoscaling.Object.DescribedObject, + Target: getMetricTargetByType(additionalAutoscaling.Object.Target), + Metric: autoscalingv2.MetricIdentifier{ + Name: additionalAutoscaling.Object.Metric.Name, + Selector: additionalAutoscaling.Object.Metric.Selector, + }, + }, + } + case autoscalingv2.ExternalMetricSourceType: + return &autoscalingv2.MetricSpec{ + Type: additionalAutoscaling.Type, + External: &autoscalingv2.ExternalMetricSource{ + Metric: autoscalingv2.MetricIdentifier{ + Name: additionalAutoscaling.External.Metric.Name, + Selector: additionalAutoscaling.External.Metric.Selector, + }, + Target: getMetricTargetByType(additionalAutoscaling.External.Target), + }, + } + default: + return nil + } +} + // TODO(sberman): see about how this can be made more elegant. Maybe create some sort of Object factory // that can better store/build all the objects we need, to reduce the amount of duplicate object lists that we // have everywhere. @@ -906,6 +1076,7 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName // serviceaccount // configmaps // secrets + // hpa objectMeta := metav1.ObjectMeta{ Name: deploymentNSName.Name, @@ -921,8 +1092,11 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName service := &corev1.Service{ ObjectMeta: objectMeta, } + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: objectMeta, + } - objects := []client.Object{deployment, daemonSet, service} + objects := []client.Object{deployment, daemonSet, service, hpa} if p.isOpenshift { role := &rbacv1.Role{ diff --git a/internal/controller/provisioner/objects_test.go b/internal/controller/provisioner/objects_test.go index 96710f8902..97ec9f58c0 100644 --- a/internal/controller/provisioner/objects_test.go +++ b/internal/controller/provisioner/objects_test.go @@ -6,6 +6,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -279,6 +280,12 @@ func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { }, Deployment: &ngfAPIv1alpha2.DeploymentSpec{ Replicas: helpers.GetPointer[int32](3), + Autoscaling: ngfAPIv1alpha2.HPASpec{ + Enabled: true, + MinReplicas: 1, + MaxReplicas: 5, + TargetMemoryUtilizationPercentage: helpers.GetPointer[int32](60), + }, Pod: ngfAPIv1alpha2.PodSpec{ TerminationGracePeriodSeconds: helpers.GetPointer[int64](25), }, @@ -301,7 +308,7 @@ func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { objects, err := provisioner.buildNginxResourceObjects(resourceName, gateway, nProxyCfg) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(objects).To(HaveLen(6)) + g.Expect(objects).To(HaveLen(7)) cmObj := objects[1] cm, ok := cmObj.(*corev1.ConfigMap) @@ -803,7 +810,7 @@ func TestBuildNginxResourceObjectsForDeletion(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) - g.Expect(objects).To(HaveLen(7)) + g.Expect(objects).To(HaveLen(8)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) @@ -825,17 +832,22 @@ func TestBuildNginxResourceObjectsForDeletion(t *testing.T) { g.Expect(ok).To(BeTrue()) validateMeta(svc, deploymentNSName.Name) - svcAcctObj := objects[3] + hpaObj := objects[3] + hpa, ok := hpaObj.(*autoscalingv2.HorizontalPodAutoscaler) + g.Expect(ok).To(BeTrue()) + validateMeta(hpa, deploymentNSName.Name) + + svcAcctObj := objects[4] svcAcct, ok := svcAcctObj.(*corev1.ServiceAccount) g.Expect(ok).To(BeTrue()) validateMeta(svcAcct, deploymentNSName.Name) - cmObj := objects[4] + cmObj := objects[5] cm, ok := cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxIncludesConfigMapNameSuffix)) - cmObj = objects[5] + cmObj = objects[6] cm, ok = cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxAgentConfigMapNameSuffix)) @@ -865,7 +877,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) - g.Expect(objects).To(HaveLen(11)) + g.Expect(objects).To(HaveLen(12)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) @@ -887,22 +899,27 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { g.Expect(ok).To(BeTrue()) validateMeta(svc, deploymentNSName.Name) - svcAcctObj := objects[3] + hpaObj := objects[3] + hpa, ok := hpaObj.(*autoscalingv2.HorizontalPodAutoscaler) + g.Expect(ok).To(BeTrue()) + validateMeta(hpa, deploymentNSName.Name) + + svcAcctObj := objects[4] svcAcct, ok := svcAcctObj.(*corev1.ServiceAccount) g.Expect(ok).To(BeTrue()) validateMeta(svcAcct, deploymentNSName.Name) - cmObj := objects[4] + cmObj := objects[5] cm, ok := cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxIncludesConfigMapNameSuffix)) - cmObj = objects[5] + cmObj = objects[6] cm, ok = cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxAgentConfigMapNameSuffix)) - secretObj := objects[6] + secretObj := objects[7] secret, ok := secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -910,7 +927,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { provisioner.cfg.AgentTLSSecretName, )) - secretObj = objects[7] + secretObj = objects[8] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -918,7 +935,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { provisioner.cfg.NginxDockerSecretNames[0], )) - secretObj = objects[8] + secretObj = objects[9] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -926,7 +943,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { provisioner.cfg.PlusUsageConfig.CASecretName, )) - secretObj = objects[9] + secretObj = objects[10] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -948,19 +965,19 @@ func TestBuildNginxResourceObjectsForDeletion_OpenShift(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) - g.Expect(objects).To(HaveLen(9)) + g.Expect(objects).To(HaveLen(10)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) g.Expect(obj.GetNamespace()).To(Equal(deploymentNSName.Namespace)) } - roleObj := objects[3] + roleObj := objects[4] role, ok := roleObj.(*rbacv1.Role) g.Expect(ok).To(BeTrue()) validateMeta(role, deploymentNSName.Name) - roleBindingObj := objects[4] + roleBindingObj := objects[5] roleBinding, ok := roleBindingObj.(*rbacv1.RoleBinding) g.Expect(ok).To(BeTrue()) validateMeta(roleBinding, deploymentNSName.Name) diff --git a/internal/controller/provisioner/provisioner_test.go b/internal/controller/provisioner/provisioner_test.go index 9102a8193f..76edf13d5d 100644 --- a/internal/controller/provisioner/provisioner_test.go +++ b/internal/controller/provisioner/provisioner_test.go @@ -7,6 +7,7 @@ import ( "github.com/go-logr/logr" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -43,6 +44,7 @@ func createScheme() *runtime.Scheme { utilruntime.Must(gatewayv1.Install(scheme)) utilruntime.Must(corev1.AddToScheme(scheme)) utilruntime.Must(appsv1.AddToScheme(scheme)) + utilruntime.Must(autoscalingv2.AddToScheme(scheme)) return scheme } diff --git a/internal/controller/provisioner/setter.go b/internal/controller/provisioner/setter.go index 59718440f4..0ffbf53d49 100644 --- a/internal/controller/provisioner/setter.go +++ b/internal/controller/provisioner/setter.go @@ -4,6 +4,7 @@ import ( "maps" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -16,6 +17,8 @@ func objectSpecSetter(object client.Object) controllerutil.MutateFn { switch obj := object.(type) { case *appsv1.Deployment: return deploymentSpecSetter(obj, obj.Spec, obj.ObjectMeta) + case *autoscalingv2.HorizontalPodAutoscaler: + return hpaSpecSetter(obj, obj.Spec, obj.ObjectMeta) case *appsv1.DaemonSet: return daemonSetSpecSetter(obj, obj.Spec, obj.ObjectMeta) case *corev1.Service: @@ -48,6 +51,19 @@ func deploymentSpecSetter( } } +func hpaSpecSetter( + hpa *autoscalingv2.HorizontalPodAutoscaler, + spec autoscalingv2.HorizontalPodAutoscalerSpec, + objectMeta metav1.ObjectMeta, +) controllerutil.MutateFn { + return func() error { + hpa.Labels = objectMeta.Labels + hpa.Annotations = objectMeta.Annotations + hpa.Spec = spec + return nil + } +} + func daemonSetSpecSetter( daemonSet *appsv1.DaemonSet, spec appsv1.DaemonSetSpec, diff --git a/internal/controller/provisioner/store.go b/internal/controller/provisioner/store.go index d089a38f9b..fd095eae5a 100644 --- a/internal/controller/provisioner/store.go +++ b/internal/controller/provisioner/store.go @@ -6,6 +6,7 @@ import ( "sync" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -20,6 +21,7 @@ import ( type NginxResources struct { Gateway *graph.Gateway Deployment metav1.ObjectMeta + HPA metav1.ObjectMeta DaemonSet metav1.ObjectMeta Service metav1.ObjectMeta ServiceAccount metav1.ObjectMeta @@ -133,6 +135,14 @@ func (s *store) registerResourceInGatewayConfig(gatewayNSName types.NamespacedNa } else { cfg.Deployment = obj.ObjectMeta } + case *autoscalingv2.HorizontalPodAutoscaler: + if cfg, ok := s.nginxResources[gatewayNSName]; !ok { + s.nginxResources[gatewayNSName] = &NginxResources{ + HPA: obj.ObjectMeta, + } + } else { + cfg.HPA = obj.ObjectMeta + } case *appsv1.DaemonSet: if cfg, ok := s.nginxResources[gatewayNSName]; !ok { s.nginxResources[gatewayNSName] = &NginxResources{ @@ -301,6 +311,10 @@ func (s *store) gatewayExistsForResource(object client.Object, nsName types.Name if resourceMatches(resources.Deployment, nsName) { return resources.Gateway } + case *autoscalingv2.HorizontalPodAutoscaler: + if resourceMatches(resources.HPA, nsName) { + return resources.Gateway + } case *appsv1.DaemonSet: if resourceMatches(resources.DaemonSet, nsName) { return resources.Gateway @@ -379,6 +393,10 @@ func (s *store) getResourceVersionForObject(gatewayNSName types.NamespacedName, if resources.Deployment.GetName() == obj.GetName() { return resources.Deployment.GetResourceVersion() } + case *autoscalingv2.HorizontalPodAutoscaler: + if resources.HPA.GetName() == obj.GetName() { + return resources.HPA.GetResourceVersion() + } case *appsv1.DaemonSet: if resources.DaemonSet.GetName() == obj.GetName() { return resources.DaemonSet.GetResourceVersion()