diff --git a/pkg/apis/v1/nodepool_conversion.go b/pkg/apis/v1/nodepool_conversion.go index 97b0258aa9..eadc3ba783 100644 --- a/pkg/apis/v1/nodepool_conversion.go +++ b/pkg/apis/v1/nodepool_conversion.go @@ -32,7 +32,7 @@ import ( ) // Convert v1 NodePool to v1beta1 NodePool -func (in *NodePool) ConvertTo(ctx context.Context, to apis.Convertible) error { +func (in *NodePool) ConvertTo(_ context.Context, to apis.Convertible) error { v1beta1NP := to.(*v1beta1.NodePool) v1beta1NP.ObjectMeta = in.ObjectMeta @@ -123,9 +123,9 @@ func (in *NodePool) ConvertFrom(ctx context.Context, v1beta1np apis.Convertible) return err } if kubeletAnnotation == "" { - in.Annotations = lo.OmitByKeys(in.Annotations, []string{KubeletCompatibilityAnnotationKey}) + delete(in.Annotations, KubeletCompatibilityAnnotationKey) } else { - in.Annotations = lo.Assign(in.Annotations, map[string]string{KubeletCompatibilityAnnotationKey: kubeletAnnotation}) + in.Annotations = lo.Assign(in.Spec.Template.Annotations, map[string]string{KubeletCompatibilityAnnotationKey: kubeletAnnotation}) } nodeClassRefAnnotation, err := json.Marshal(v1beta1NP.Spec.Template.Spec.NodeClassRef) if err != nil { diff --git a/pkg/controllers/provisioning/scheduling/nodeclaimtemplate.go b/pkg/controllers/provisioning/scheduling/nodeclaimtemplate.go index 73e462076f..2b7b99e7d9 100644 --- a/pkg/controllers/provisioning/scheduling/nodeclaimtemplate.go +++ b/pkg/controllers/provisioning/scheduling/nodeclaimtemplate.go @@ -50,6 +50,11 @@ func NewNodeClaimTemplate(nodePool *v1.NodePool) *NodeClaimTemplate { NodePoolName: nodePool.Name, Requirements: scheduling.NewRequirements(), } + if v, ok := nodePool.Annotations[v1.KubeletCompatibilityAnnotationKey]; ok { + nct.Annotations = lo.Assign(nct.Annotations, map[string]string{ + v1.KubeletCompatibilityAnnotationKey: v, + }) + } nct.Labels = lo.Assign(nct.Labels, map[string]string{v1.NodePoolLabelKey: nodePool.Name}) nct.Requirements.Add(scheduling.NewNodeSelectorRequirementsWithMinValues(nct.Spec.Requirements...).Values()...) nct.Requirements.Add(scheduling.NewLabelRequirements(nct.Labels).Values()...) diff --git a/pkg/controllers/provisioning/suite_test.go b/pkg/controllers/provisioning/suite_test.go index 19db025a17..1d0530b5cc 100644 --- a/pkg/controllers/provisioning/suite_test.go +++ b/pkg/controllers/provisioning/suite_test.go @@ -18,10 +18,12 @@ package provisioning_test import ( "context" + "encoding/json" "fmt" "testing" "time" + "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/test/v1alpha1" . "github.com/onsi/ginkgo/v2" @@ -248,6 +250,38 @@ var _ = Describe("Provisioning", func() { ExpectScheduled(ctx, env.Client, pod) } }) + It("should add the NodePool's karpenter.sh/v1beta1-kubelet-conversion annotation when scheduling", func() { + nodePool := test.NodePool() + kubeletConfig := v1beta1.KubeletConfiguration{ + ClusterDNS: []string{"test-cluster-dns"}, + MaxPods: lo.ToPtr(int32(9383)), + PodsPerCore: lo.ToPtr(int32(9334283)), + SystemReserved: map[string]string{"system-key": "reserved"}, + KubeReserved: map[string]string{"kube-key": "reserved"}, + EvictionHard: map[string]string{"eviction-key": "eviction"}, + EvictionSoft: map[string]string{"eviction-key": "eviction"}, + EvictionSoftGracePeriod: map[string]metav1.Duration{"test-soft-grace": {Duration: time.Hour}}, + EvictionMaxPodGracePeriod: lo.ToPtr(int32(382902)), + ImageGCHighThresholdPercent: lo.ToPtr(int32(382902)), + CPUCFSQuota: lo.ToPtr(false), + } + raw, err := json.Marshal(kubeletConfig) + Expect(err).ToNot(HaveOccurred()) + nodePool.Annotations = lo.Assign(nodePool.Annotations, map[string]string{ + v1.KubeletCompatibilityAnnotationKey: string(raw), + }) + + ExpectApplied(ctx, env.Client, nodePool) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + + nodeClaims := &v1.NodeClaimList{} + Expect(env.Client.List(ctx, nodeClaims)).To(Succeed()) + Expect(nodeClaims.Items).To(HaveLen(1)) + + Expect(nodeClaims.Items[0].Annotations).To(HaveKeyWithValue(v1.KubeletCompatibilityAnnotationKey, nodePool.Annotations[v1.KubeletCompatibilityAnnotationKey])) + ExpectScheduled(ctx, env.Client, pod) + }) It("should schedule all pods on one inflight node when node is in deleting state", func() { nodePool := test.NodePool() its, err := cloudProvider.GetInstanceTypes(ctx, nodePool)