From 8ef1778d7aeec8ffc6a1bbee3d87fda3eb309c09 Mon Sep 17 00:00:00 2001 From: Meet Vasani Date: Mon, 11 Mar 2024 01:00:07 +0530 Subject: [PATCH] Add Support for HPA Object (#28) * chore: add initial struct for HPA * chore: add default min replica * chore: add hpa to main nimble Object * chore: remove replicas field from manifest reason behind is that when using with HPA it is ideal that we don't define replicas * docs: add example for hpa usecase * feat: add controller for hpa * chore: add requests field for hpa usecase * chore: update mods * feat: add hpa controller to main * chore: update crd --- crd/nimble.ivaltryek.github.com.yaml | 49 ++++++++- examples/deployment-service-hpa.yaml | 27 +++++ examples/deployment-with-service.yaml | 2 + src/controllers/dpcontroller.rs | 1 - src/controllers/hpacontroller.rs | 147 ++++++++++++++++++++++++++ src/controllers/mod.rs | 1 + src/crds/deploymentspec.rs | 13 --- src/crds/hpaspec.rs | 50 +++++++++ src/crds/mod.rs | 1 + src/crds/nimble.rs | 6 +- src/main.rs | 4 +- src/transformers/hpa.rs | 28 +++++ src/transformers/mod.rs | 1 + 13 files changed, 310 insertions(+), 20 deletions(-) create mode 100644 examples/deployment-service-hpa.yaml create mode 100644 src/controllers/hpacontroller.rs create mode 100644 src/crds/hpaspec.rs create mode 100644 src/transformers/hpa.rs diff --git a/crd/nimble.ivaltryek.github.com.yaml b/crd/nimble.ivaltryek.github.com.yaml index b9d49e0..5f21ad2 100644 --- a/crd/nimble.ivaltryek.github.com.yaml +++ b/crd/nimble.ivaltryek.github.com.yaml @@ -21,6 +21,7 @@ spec: spec: properties: deployment: + description: Spec for Deployment Object properties: annotations: additionalProperties: @@ -278,16 +279,56 @@ spec: type: string description: Labels to be applied to the deployment and its pods. type: object - replicas: + required: + - containers + - labels + type: object + hpa: + description: Spec for Autoscaling (HPA) Object + nullable: true + properties: + annotations: + additionalProperties: + type: string + default: + app.kubernetes.io/managed-by: kube-nimble + description: Annotations to be applied to the HPA object + nullable: true + type: object + max: + description: maxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. It cannot be less that minReplicas. + format: int32 + type: integer + min: default: 1 - description: Number of desired replicas for the deployment. + description: minReplicas is the lower limit for the number of replicas to which the autoscaler can scale down. It defaults to 1 pod. minReplicas is allowed to be 0 if the alpha feature gate HPAScaleToZero is enabled and at least one Object or External metric is configured. Scaling is active as long as at least one metric value is available. format: int32 + nullable: true type: integer + resourcePolicy: + description: resource refers to a resource metric (such as those specified in requests and limits) known to Kubernetes describing each pod in the current scale target (e.g. CPU or memory). + nullable: true + properties: + avgUtil: + description: avgUtil is the target value of the average of the resource metric across all relevant pods, represented as a percentage of the requested value of the resource for the pods. Currently only valid for Resource metric source type + format: int32 + nullable: true + type: integer + name: + description: name is the name of the resource in question. + type: string + type: + description: type represents whether the metric type is Utilization, Value, or AverageValue + type: string + required: + - name + - type + type: object required: - - containers - - labels + - max type: object service: + description: Spec for Service Object nullable: true properties: annotations: diff --git a/examples/deployment-service-hpa.yaml b/examples/deployment-service-hpa.yaml new file mode 100644 index 0000000..c3b1120 --- /dev/null +++ b/examples/deployment-service-hpa.yaml @@ -0,0 +1,27 @@ +apiVersion: ivaltryek.github.com/v1 +kind: Nimble +metadata: + name: demo-deployment-service-hpa + namespace: test +spec: + deployment: + containers: + - image: nginx:stable + name: nginx-stable + requests: + cpu: 50m + labels: + test: hpa + service: + ports: + - name: http + port: 80 + targetPort: 80 + hpa: + min: 2 + max: 4 + resourcePolicy: + name: cpu + type: Utilization + avgUtil: 30 + diff --git a/examples/deployment-with-service.yaml b/examples/deployment-with-service.yaml index 333b14a..137bc67 100644 --- a/examples/deployment-with-service.yaml +++ b/examples/deployment-with-service.yaml @@ -8,6 +8,8 @@ spec: containers: - image: nginx:stable name: nginx-stable + requests: + cpu: 50m labels: app: nginx env: test diff --git a/src/controllers/dpcontroller.rs b/src/controllers/dpcontroller.rs index ac869d0..37c931e 100644 --- a/src/controllers/dpcontroller.rs +++ b/src/controllers/dpcontroller.rs @@ -60,7 +60,6 @@ pub async fn reconcile(nimble: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result { + match nimble.spec.hpa.clone() { + Some(hpa_spec) => { + let client = &ctx.client; + + let oref = nimble.controller_owner_ref(&()).unwrap(); + + let hpa: HorizontalPodAutoscaler = HorizontalPodAutoscaler { + metadata: ObjectMeta { + annotations: hpa_spec.annotations.clone(), + owner_references: Some(vec![oref]), + name: nimble.metadata.name.clone(), + ..ObjectMeta::default() + }, + spec: Some(HorizontalPodAutoscalerSpec { + max_replicas: hpa_spec.max, + min_replicas: hpa_spec.min, + scale_target_ref: CrossVersionObjectReference { + api_version: Some("apps/v1".to_owned()), + kind: "Deployment".to_owned(), + name: nimble.metadata.name.clone().unwrap(), + }, + metrics: transform_metrics(nimble.spec.hpa.clone()), + ..HorizontalPodAutoscalerSpec::default() + }), + ..HorizontalPodAutoscaler::default() + }; + + let hpa_api = Api::::namespaced( + client.clone(), + nimble + .metadata + .namespace + .as_ref() + .ok_or_else(|| Error::MissingObjectKey(".metadata.namespace"))?, + ); + + hpa_api + .patch( + hpa.metadata + .name + .as_ref() + .ok_or_else(|| Error::MissingObjectKey(".metadata.name"))?, + &PatchParams::apply("nimble.ivaltryek.github.com"), + &Patch::Apply(&hpa), + ) + .await + .map_err(Error::NimbleObjectCreationFailed)?; + + DOES_HPA_EXIST.store(true, Ordering::Relaxed); + + Ok(Action::requeue(Duration::from_secs(30))) + } + _ => { + DOES_HPA_EXIST.store(false, Ordering::Relaxed); + Ok(Action::await_change()) + } + } +} + +/** + * Starts the main loop for the Nimble HPA controller. + * + * This function initiates the main event loop for the Nimble controller, responsible for monitoring and reconciling Nimble resources in the Kubernetes cluster. + * + * Args: + * - crd_api (Api): Reference to the Kubernetes API client for Nimble resources. + * - context (Arc): Reference-counted handle to the controller context data. + * + * Returns: + * - Future: Represents the completion of the controller loop. + * + * Process: + * 1. Creates a new controller instance using the provided API client and default configuration. + * 2. Configures the controller to shut down gracefully on receiving specific signals. + * 3. Starts the controller loop, running the `reconcile` function for each Nimble resource change it detects. + * 4. Within the loop, handles reconciliation results: + * - On success: logs a message with resource information. + * - On error: logs an error message with details. + * 5. Waits for the loop to complete. + */ +pub async fn run_hpa_controller(crd_api: Api, context: Arc) { + Controller::new(crd_api.clone(), Config::default()) + .shutdown_on_signal() + .run(reconcile, error_policy, context) + .for_each(|reconcilation_result| async move { + match reconcilation_result { + Ok((nimble_resource, _)) => { + // Log the reconciliation message only if service field exist in object manifest. + if DOES_HPA_EXIST.load(Ordering::Relaxed) { + info!(msg = "HPA reconciliation successful.", + resource_name = ?nimble_resource.name, + namespace = ?nimble_resource.namespace.unwrap(), + ); + } + } + Err(reconciliation_err) => { + error!("Service reconciliation error: {:?}", reconciliation_err) + } + } + }) + .await; +} diff --git a/src/controllers/mod.rs b/src/controllers/mod.rs index 597fcf5..5b7b177 100644 --- a/src/controllers/mod.rs +++ b/src/controllers/mod.rs @@ -1,2 +1,3 @@ pub mod dpcontroller; +pub mod hpacontroller; pub mod servicecontroller; diff --git a/src/crds/deploymentspec.rs b/src/crds/deploymentspec.rs index 527d326..2148fe2 100644 --- a/src/crds/deploymentspec.rs +++ b/src/crds/deploymentspec.rs @@ -7,9 +7,6 @@ use serde::{Deserialize, Serialize}; pub struct DeploySpec { #[doc = "Containers to run in the deployment."] pub containers: Vec, - #[doc = "Number of desired replicas for the deployment."] - #[serde(default = "default_replicas")] - pub replicas: i32, #[doc = "Labels to be applied to the deployment and its pods."] pub labels: BTreeMap, #[doc = "Annotations to be applied to the deployment and its pods."] @@ -132,16 +129,6 @@ pub struct EnvFromSpec { pub secret_ref: Option, } -/** - * This function returns the default value for the number of replicas. - * In this specific case, the default is set to 1. - * - * You can customize this value by modifying the function body. - */ -pub fn default_replicas() -> i32 { - 1 -} - /* This function creates a default `Option>` containing a single key-value pair: * - "app.kubernetes.io/managed-by": "kube-nimble" * diff --git a/src/crds/hpaspec.rs b/src/crds/hpaspec.rs new file mode 100644 index 0000000..6d128c3 --- /dev/null +++ b/src/crds/hpaspec.rs @@ -0,0 +1,50 @@ +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::deploymentspec::default_annotations; + +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, JsonSchema)] +pub struct HPASpec { + #[doc = "Annotations to be applied to the HPA object"] + #[serde(default = "default_hpa_annotations")] + pub annotations: Option>, + #[doc = "maxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. + It cannot be less that minReplicas."] + pub max: i32, + #[doc = "minReplicas is the lower limit for the number of replicas to which the autoscaler can scale down. + It defaults to 1 pod. + minReplicas is allowed to be 0 if the alpha feature gate HPAScaleToZero is enabled and at least one Object or External metric is configured. + Scaling is active as long as at least one metric value is available."] + #[serde(default = "default_min_replicas")] + pub min: Option, + #[doc = "resource refers to a resource metric (such as those specified in requests and limits) + known to Kubernetes describing each pod in the current scale target (e.g. CPU or memory)."] + #[serde(rename = "resourcePolicy")] + pub resource_policy: Option, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, JsonSchema)] +pub struct ResourceMetricSpec { + #[doc = "name is the name of the resource in question."] + pub name: String, + #[doc = "type represents whether the metric type is Utilization, Value, or AverageValue"] + #[serde(rename = "type")] + pub type_: String, + #[doc = "avgUtil is the target value of the average of the resource metric across all relevant pods, + represented as a percentage of the requested value of the resource for the pods. + Currently only valid for Resource metric source type"] + #[serde(rename = "avgUtil")] + pub average_utilization: Option, +} + +// Return default annotations to applied to an object. +fn default_hpa_annotations() -> Option> { + default_annotations() +} + +// Return default min replica value. +fn default_min_replicas() -> Option { + Some(1) +} diff --git a/src/crds/mod.rs b/src/crds/mod.rs index 4a020cd..ef3a68e 100644 --- a/src/crds/mod.rs +++ b/src/crds/mod.rs @@ -1,3 +1,4 @@ pub mod deploymentspec; +pub mod hpaspec; pub mod nimble; pub mod servicespec; diff --git a/src/crds/nimble.rs b/src/crds/nimble.rs index 7e36ef0..3e453b2 100644 --- a/src/crds/nimble.rs +++ b/src/crds/nimble.rs @@ -1,7 +1,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use super::{deploymentspec::DeploySpec, servicespec::SvcSpec}; +use super::{deploymentspec::DeploySpec, hpaspec::HPASpec, servicespec::SvcSpec}; #[derive(kube::CustomResource, Serialize, Deserialize, Debug, PartialEq, Clone, JsonSchema)] #[kube( @@ -14,6 +14,10 @@ use super::{deploymentspec::DeploySpec, servicespec::SvcSpec}; )] pub struct NimbleSpec { + #[doc = "Spec for Deployment Object"] pub deployment: DeploySpec, + #[doc = "Spec for Service Object"] pub service: Option, + #[doc = "Spec for Autoscaling (HPA) Object"] + pub hpa: Option, } diff --git a/src/main.rs b/src/main.rs index 17a96f5..130bc64 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ use tracing::info; use crate::common::client::ContextData; use crate::controllers::dpcontroller::run_dp_controller; +use crate::controllers::hpacontroller::run_hpa_controller; use crate::controllers::servicecontroller::run_svc_controller; use crate::crds::nimble::Nimble; @@ -23,9 +24,10 @@ async fn main() { info!("starting nimble controller"); - let (_, _) = futures::join!( + let (_, _, _) = futures::join!( run_dp_controller(crd_api.clone(), context.clone()), run_svc_controller(crd_api.clone(), context.clone()), + run_hpa_controller(crd_api.clone(), context.clone()) ); info!("controller terminated"); diff --git a/src/transformers/hpa.rs b/src/transformers/hpa.rs new file mode 100644 index 0000000..d0a3ae0 --- /dev/null +++ b/src/transformers/hpa.rs @@ -0,0 +1,28 @@ +use k8s_openapi::api::autoscaling::v2::{MetricSpec, MetricTarget, ResourceMetricSource}; + +use crate::crds::hpaspec::HPASpec; + +pub fn transform_metrics(hpa_spec: Option) -> Option> { + let mut metric_spec_vec = Vec::new(); + + match hpa_spec.clone() { + Some(hpa) => { + if let Some(resource_policy) = hpa.resource_policy { + metric_spec_vec.push(MetricSpec { + type_: "Resource".to_owned(), + resource: Some(ResourceMetricSource { + name: resource_policy.name, + target: MetricTarget { + average_utilization: resource_policy.average_utilization, + type_: resource_policy.type_, + ..MetricTarget::default() + }, + }), + ..MetricSpec::default() + }); + } + Some(metric_spec_vec) + } + _ => None, + } +} diff --git a/src/transformers/mod.rs b/src/transformers/mod.rs index 03bf4b5..1e8b2d9 100644 --- a/src/transformers/mod.rs +++ b/src/transformers/mod.rs @@ -1,2 +1,3 @@ pub mod deployment; +pub mod hpa; pub mod service;