From ecf74e22de732ba32f0294b334b11e7921f860f0 Mon Sep 17 00:00:00 2001 From: Xavier Duthil Date: Thu, 9 Mar 2023 11:51:19 +0100 Subject: [PATCH] fix(ovhcloud): Add support for nodepool templating Should fix the autoscaler being unable to scale up an empty nodepool with specific template, because the template was not processed when setting up the upscale simulation (with the following error: "node(s) didn't match Pod's node affinity/selector") Signed-off-by: Xavier Duthil --- .../ovhcloud/ovh_cloud_node_group.go | 18 +++++-- .../ovhcloud/ovh_cloud_node_group_test.go | 49 ++++++++++++++++++- .../cloudprovider/ovhcloud/sdk/nodepool.go | 15 ++++++ 3 files changed, 75 insertions(+), 7 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/ovhcloud/ovh_cloud_node_group.go b/cluster-autoscaler/cloudprovider/ovhcloud/ovh_cloud_node_group.go index 2ae6eec19ecf..64144bd77634 100644 --- a/cluster-autoscaler/cloudprovider/ovhcloud/ovh_cloud_node_group.go +++ b/cluster-autoscaler/cloudprovider/ovhcloud/ovh_cloud_node_group.go @@ -206,18 +206,26 @@ func (ng *NodeGroup) TemplateNodeInfo() (*schedulerframework.NodeInfo, error) { // Forge node template in a node group node := &apiv1.Node{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-node-%d", ng.Id(), rand.Int63()), - Labels: map[string]string{ - NodePoolLabel: ng.Id(), - }, + Name: fmt.Sprintf("%s-node-%d", ng.Id(), rand.Int63()), + Labels: ng.Template.Metadata.Labels, + Annotations: ng.Template.Metadata.Annotations, + Finalizers: ng.Template.Metadata.Finalizers, + }, + Spec: apiv1.NodeSpec{ + Taints: ng.Template.Spec.Taints, }, - Spec: apiv1.NodeSpec{}, Status: apiv1.NodeStatus{ Capacity: apiv1.ResourceList{}, Conditions: cloudprovider.BuildReadyConditions(), }, } + // Add the nodepool label + if node.ObjectMeta.Labels == nil { + node.ObjectMeta.Labels = make(map[string]string) + } + node.ObjectMeta.Labels[NodePoolLabel] = ng.Id() + flavor, err := ng.Manager.getFlavorByName(ng.Flavor) if err != nil { return nil, fmt.Errorf("failed to get specs for flavor %q: %w", ng.Flavor, err) diff --git a/cluster-autoscaler/cloudprovider/ovhcloud/ovh_cloud_node_group_test.go b/cluster-autoscaler/cloudprovider/ovhcloud/ovh_cloud_node_group_test.go index 7efb85356d04..df8a4ab0f917 100644 --- a/cluster-autoscaler/cloudprovider/ovhcloud/ovh_cloud_node_group_test.go +++ b/cluster-autoscaler/cloudprovider/ovhcloud/ovh_cloud_node_group_test.go @@ -379,7 +379,10 @@ func TestOVHCloudNodeGroup_TemplateNodeInfo(t *testing.T) { assert.NotNil(t, node) assert.Contains(t, node.ObjectMeta.Name, fmt.Sprintf("%s-node-", ng.Id())) - assert.Equal(t, ng.Id(), node.Labels["nodepool"]) + assert.Equal(t, map[string]string{"nodepool": ng.Id()}, node.Labels) + assert.Equal(t, map[string]string(nil), node.Annotations) + assert.Equal(t, []string(nil), node.Finalizers) + assert.Equal(t, []v1.Taint(nil), node.Spec.Taints) assert.Equal(t, *resource.NewQuantity(110, resource.DecimalSI), node.Status.Capacity[apiv1.ResourcePods]) assert.Equal(t, *resource.NewQuantity(2, resource.DecimalSI), node.Status.Capacity[apiv1.ResourceCPU]) @@ -396,7 +399,49 @@ func TestOVHCloudNodeGroup_TemplateNodeInfo(t *testing.T) { assert.NotNil(t, node) assert.Contains(t, node.ObjectMeta.Name, fmt.Sprintf("%s-node-", ng.Id())) - assert.Equal(t, ng.Id(), node.Labels["nodepool"]) + assert.Equal(t, map[string]string{"nodepool": ng.Id()}, node.Labels) + assert.Equal(t, map[string]string(nil), node.Annotations) + assert.Equal(t, []string(nil), node.Finalizers) + assert.Equal(t, []v1.Taint(nil), node.Spec.Taints) + + assert.Equal(t, *resource.NewQuantity(110, resource.DecimalSI), node.Status.Capacity[apiv1.ResourcePods]) + assert.Equal(t, *resource.NewQuantity(8, resource.DecimalSI), node.Status.Capacity[apiv1.ResourceCPU]) + assert.Equal(t, *resource.NewQuantity(1, resource.DecimalSI), node.Status.Capacity[gpu.ResourceNvidiaGPU]) + assert.Equal(t, *resource.NewQuantity(48318382080, resource.DecimalSI), node.Status.Capacity[apiv1.ResourceMemory]) + }) + + t.Run("template for b2-7 flavor with node pool templates", func(t *testing.T) { + ng := newTestNodeGroup(t, "t1-45") + + // Setup node pool templates + ng.Template.Metadata.Labels = map[string]string{ + "label1": "labelValue1", + } + ng.Template.Metadata.Annotations = map[string]string{ + "annotation1": "annotationValue1", + } + ng.Template.Metadata.Finalizers = []string{ + "finalizer1", + } + ng.Template.Spec.Taints = []v1.Taint{ + { + Key: "taintKey1", + Value: "taintValue1", + Effect: "taintEffect1", + }, + } + + template, err := ng.TemplateNodeInfo() + assert.NoError(t, err) + + node := template.Node() + assert.NotNil(t, node) + + assert.Contains(t, node.ObjectMeta.Name, fmt.Sprintf("%s-node-", ng.Id())) + assert.Equal(t, map[string]string{"nodepool": ng.Id(), "label1": "labelValue1"}, node.Labels) + assert.Equal(t, ng.Template.Metadata.Annotations, node.Annotations) + assert.Equal(t, ng.Template.Metadata.Finalizers, node.Finalizers) + assert.Equal(t, ng.Template.Spec.Taints, node.Spec.Taints) assert.Equal(t, *resource.NewQuantity(110, resource.DecimalSI), node.Status.Capacity[apiv1.ResourcePods]) assert.Equal(t, *resource.NewQuantity(8, resource.DecimalSI), node.Status.Capacity[apiv1.ResourceCPU]) diff --git a/cluster-autoscaler/cloudprovider/ovhcloud/sdk/nodepool.go b/cluster-autoscaler/cloudprovider/ovhcloud/sdk/nodepool.go index d9f935cb062e..589acdd7a165 100644 --- a/cluster-autoscaler/cloudprovider/ovhcloud/sdk/nodepool.go +++ b/cluster-autoscaler/cloudprovider/ovhcloud/sdk/nodepool.go @@ -20,6 +20,8 @@ import ( "context" "fmt" "time" + + v1 "k8s.io/api/core/v1" ) // NodePool defines the nodes group deployed on OVHcloud @@ -45,6 +47,19 @@ type NodePool struct { Autoscaling *NodePoolAutoscaling `json:"autoscaling,omitempty"` + Template struct { + Metadata struct { + Labels map[string]string `json:"labels"` + Annotations map[string]string `json:"annotations"` + Finalizers []string `json:"finalizers"` + } `json:"metadata"` + + Spec struct { + Unschedulable bool `json:"unschedulable"` + Taints []v1.Taint `json:"taints"` + } `json:"spec"` + } `json:"template"` + CreatedAt time.Time `json:"createdAt"` UpdatedAt time.Time `json:"updatedAt"` }