feat: Add support for minValues in requirement for scheduling, consolidation layer and corresponding API changes.

kwok/cloudprovider/cloudprovider.go

@@ -134,12 +134,12 @@ func (c CloudProvider) toNode(nodeClaim *v1beta1.NodeClaim) (*v1.Node, error) {
 	newName = fmt.Sprintf("%s-%d", newName, rand.Uint32())
 
 	capacityType := v1beta1.CapacityTypeOnDemand
-	requirements := scheduling.NewNodeSelectorRequirements(nodeClaim.
+	requirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.
 		Spec.Requirements...)
 	if requirements.Get(v1beta1.CapacityTypeLabelKey).Has(v1beta1.CapacityTypeSpot) {
 		capacityType = v1beta1.CapacityTypeSpot
 	}
-	req, found := lo.Find(nodeClaim.Spec.Requirements, func(req v1.NodeSelectorRequirement) bool {
+	req, found := lo.Find(nodeClaim.Spec.Requirements, func(req v1beta1.NodeSelectorRequirementWithMinValues) bool {
 		return req.Key == v1.LabelInstanceTypeStable
 	})
 	if !found {

pkg/apis/crds/karpenter.sh_nodeclaims.yaml

@@ -208,8 +208,8 @@ spec:
                   description: Requirements are layered with GetLabels and applied to every node.
                   items:
                     description: |-
-                      A node selector requirement is a selector that contains values, a key, and an operator
-                      that relates the key and values.
+                      A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values
+                      and minValues that represent the requirement to have at least that many values.
                     properties:
                       key:
                         description: The label key that the selector applies to.
@@ -225,6 +225,13 @@ spec:
                             rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh")
                           - message: label "kubernetes.io/hostname" is restricted
                             rule: self != "kubernetes.io/hostname"
+                      minValues:
+                        description: |-
+                          This field is ALPHA and can be dropped or replaced at any time
+                          MinValues is the minimum number of unique values required to define the flexibility of the specific requirement.
+                        maximum: 50
+                        minimum: 1
+                        type: integer
                       operator:
                         description: |-
                           Represents a key's relationship to a set of values.
@@ -260,6 +267,8 @@ spec:
                       rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)'
                     - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value
                       rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)'
+                    - message: requirements with 'minValues' must have at least that many values specified in the 'values' field
+                      rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)'
                 resources:
                   description: Resources models the resource requirements for the NodeClaim to launch
                   properties:

pkg/apis/crds/karpenter.sh_nodepools.yaml

@@ -332,8 +332,8 @@ spec:
                           description: Requirements are layered with GetLabels and applied to every node.
                           items:
                             description: |-
-                              A node selector requirement is a selector that contains values, a key, and an operator
-                              that relates the key and values.
+                              A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values
+                              and minValues that represent the requirement to have at least that many values.
                             properties:
                               key:
                                 description: The label key that the selector applies to.
@@ -351,6 +351,13 @@ spec:
                                     rule: self != "karpenter.sh/nodepool"
                                   - message: label "kubernetes.io/hostname" is restricted
                                     rule: self != "kubernetes.io/hostname"
+                              minValues:
+                                description: |-
+                                  This field is ALPHA and can be dropped or replaced at any time
+                                  MinValues is the minimum number of unique values required to define the flexibility of the specific requirement.
+                                maximum: 50
+                                minimum: 1
+                                type: integer
                               operator:
                                 description: |-
                                   Represents a key's relationship to a set of values.
@@ -386,6 +393,8 @@ spec:
                               rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)'
                             - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value
                               rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)'
+                            - message: requirements with 'minValues' must have at least that many values specified in the 'values' field
+                              rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)'
                         resources:
                           description: Resources models the resource requirements for the NodeClaim to launch
                           properties:

pkg/apis/v1beta1/nodeclaim.go

@@ -36,9 +36,10 @@ type NodeClaimSpec struct {
 	// Requirements are layered with GetLabels and applied to every node.
 	// +kubebuilder:validation:XValidation:message="requirements with operator 'In' must have a value defined",rule="self.all(x, x.operator == 'In' ? x.values.size() != 0 : true)"
 	// +kubebuilder:validation:XValidation:message="requirements operator 'Gt' or 'Lt' must have a single positive integer value",rule="self.all(x, (x.operator == 'Gt' || x.operator == 'Lt') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)"
+	// +kubebuilder:validation:XValidation:message="requirements with 'minValues' must have at least that many values specified in the 'values' field",rule="self.all(x, (x.operator == 'In' && has(x.minValues)) ? x.values.size() >= x.minValues : true)"
 	// +kubebuilder:validation:MaxItems:=30
 	// +required
-	Requirements []v1.NodeSelectorRequirement `json:"requirements" hash:"ignore"`
+	Requirements []NodeSelectorRequirementWithMinValues `json:"requirements" hash:"ignore"`
 	// Resources models the resource requirements for the NodeClaim to launch
 	// +optional
 	Resources ResourceRequirements `json:"resources,omitempty" hash:"ignore"`
@@ -54,6 +55,18 @@ type NodeClaimSpec struct {
 	NodeClassRef *NodeClassReference `json:"nodeClassRef"`
 }
 
+// A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values
+// and minValues that represent the requirement to have at least that many values.
+type NodeSelectorRequirementWithMinValues struct {
+	v1.NodeSelectorRequirement `json:",inline"`
+	// This field is ALPHA and can be dropped or replaced at any time
+	// MinValues is the minimum number of unique values required to define the flexibility of the specific requirement.
+	// +kubebuilder:validation:Minimum:=1
+	// +kubebuilder:validation:Maximum:=50
+	// +optional
+	MinValues *int `json:"minValues,omitempty"`
+}
+
 // ResourceRequirements models the required resources for the NodeClaim to launch
 // Ths will eventually be transformed into v1.ResourceRequirements when we support resources.limits
 type ResourceRequirements struct {

pkg/apis/v1beta1/nodeclaim_validation.go

@@ -141,7 +141,7 @@ func (in *NodeClaimSpec) validateRequirements() (errs *apis.FieldError) {
 	return errs
 }
 
-func ValidateRequirement(requirement v1.NodeSelectorRequirement) error { //nolint:gocyclo
+func ValidateRequirement(requirement NodeSelectorRequirementWithMinValues) error { //nolint:gocyclo
 	var errs error
 	if normalized, ok := NormalizedLabels[requirement.Key]; ok {
 		requirement.Key = normalized
@@ -163,6 +163,11 @@ func ValidateRequirement(requirement v1.NodeSelectorRequirement) error { //nolin
 	if requirement.Operator == v1.NodeSelectorOpIn && len(requirement.Values) == 0 {
 		errs = multierr.Append(errs, fmt.Errorf("key %s with operator %s must have a value defined", requirement.Key, requirement.Operator))
 	}
+
+	if requirement.Operator == v1.NodeSelectorOpIn && requirement.MinValues != nil && len(requirement.Values) < lo.FromPtr(requirement.MinValues) {
+		errs = multierr.Append(errs, fmt.Errorf("key %s with operator %s must have at least minimum number of values defined in 'values' field", requirement.Key, requirement.Operator))
+	}
+
 	if requirement.Operator == v1.NodeSelectorOpGt || requirement.Operator == v1.NodeSelectorOpLt {
 		if len(requirement.Values) != 1 {
 			errs = multierr.Append(errs, fmt.Errorf("key %s with operator %s must have a single positive integer value", requirement.Key, requirement.Operator))

pkg/apis/v1beta1/nodeclaim_validation_cel_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1beta1_test
 
 import (
+	"strconv"
 	"strings"
 	"time"
 
@@ -25,6 +26,7 @@ import (
 	"github.com/Pallinder/go-randomdata"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/samber/lo"
 	"knative.dev/pkg/ptr"
 
 	. "sigs.k8s.io/karpenter/pkg/apis/v1beta1"
@@ -48,10 +50,12 @@ var _ = Describe("Validation", func() {
 					Kind: "NodeClaim",
 					Name: "default",
 				},
-				Requirements: []v1.NodeSelectorRequirement{
+				Requirements: []NodeSelectorRequirementWithMinValues{
 					{
-						Key:      CapacityTypeLabelKey,
-						Operator: v1.NodeSelectorOpExists,
+						NodeSelectorRequirement: v1.NodeSelectorRequirement{
+							Key:      CapacityTypeLabelKey,
+							Operator: v1.NodeSelectorOpExists,
+						},
 					},
 				},
 			},
@@ -94,36 +98,36 @@ var _ = Describe("Validation", func() {
 	})
 	Context("Requirements", func() {
 		It("should allow supported ops", func() {
-			nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}},
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"1"}},
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"1"}},
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn},
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpExists},
+			nodeClaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}},
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"1"}}},
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"1"}}},
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn}},
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpExists}},
 			}
 			Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
 		})
 		It("should fail for unsupported ops", func() {
 			for _, op := range []v1.NodeSelectorOperator{"unknown"} {
-				nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
-					{Key: v1.LabelTopologyZone, Operator: op, Values: []string{"test"}},
+				nodeClaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{
+					{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: op, Values: []string{"test"}}},
 				}
 				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
 			}
 		})
 		It("should fail for restricted domains", func() {
 			for label := range RestrictedLabelDomains {
-				nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
-					{Key: label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}},
+				nodeClaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{
+					{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}},
 				}
 				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
 			}
 		})
 		It("should allow restricted domains exceptions", func() {
 			oldNodeClaim := nodeClaim.DeepCopy()
 			for label := range LabelDomainExceptions {
-				nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
-					{Key: label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}},
+				nodeClaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{
+					{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}},
 				}
 				Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
 				Expect(env.Client.Delete(ctx, nodeClaim)).To(Succeed())
@@ -133,8 +137,8 @@ var _ = Describe("Validation", func() {
 		It("should allow restricted subdomains exceptions", func() {
 			oldNodeClaim := nodeClaim.DeepCopy()
 			for label := range LabelDomainExceptions {
-				nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
-					{Key: "subdomain." + label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}},
+				nodeClaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{
+					{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "subdomain." + label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}},
 				}
 				Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
 				Expect(env.Client.Delete(ctx, nodeClaim)).To(Succeed())
@@ -144,40 +148,74 @@ var _ = Describe("Validation", func() {
 		It("should allow well known label exceptions", func() {
 			oldNodeClaim := nodeClaim.DeepCopy()
 			for label := range WellKnownLabels.Difference(sets.New(NodePoolLabelKey)) {
-				nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
-					{Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}},
+				nodeClaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{
+					{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}},
 				}
 				Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
 				Expect(env.Client.Delete(ctx, nodeClaim)).To(Succeed())
 				nodeClaim = oldNodeClaim.DeepCopy()
 			}
 		})
 		It("should allow non-empty set after removing overlapped value", func() {
-			nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test", "foo"}},
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test", "bar"}},
+			nodeClaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test", "foo"}}},
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test", "bar"}}},
 			}
 			Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
 		})
 		It("should allow empty requirements", func() {
-			nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{}
+			nodeClaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{}
 			Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
 		})
 		It("should fail with invalid GT or LT values", func() {
-			for _, requirement := range []v1.NodeSelectorRequirement{
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{}},
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"1", "2"}},
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"a"}},
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"-1"}},
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{}},
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"1", "2"}},
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"a"}},
-				{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"-1"}},
+			for _, requirement := range []NodeSelectorRequirementWithMinValues{
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{}}},
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"1", "2"}}},
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"a"}}},
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpGt, Values: []string{"-1"}}},
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{}}},
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"1", "2"}}},
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"a"}}},
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpLt, Values: []string{"-1"}}},
 			} {
-				nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{requirement}
+				nodeClaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{requirement}
 				Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
 			}
 		})
+		It("should error when minValues is negative", func() {
+			nodeClaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"insance-type-1"}}, MinValues: lo.ToPtr(-1)},
+			}
+			Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+		})
+		It("should error when minValues is zero", func() {
+			nodeClaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"insance-type-1"}}, MinValues: lo.ToPtr(0)},
+			}
+			Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+		})
+		It("should error when minValues is more than 50", func() {
+			nodeClaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpExists}, MinValues: lo.ToPtr(51)},
+			}
+			Expect(env.Client.Create(ctx, nodeClaim)).ToNot(Succeed())
+		})
+		It("should allow more than 50 values if minValues is not specified.", func() {
+			var instanceTypes []string
+			for i := 0; i < 90; i++ {
+				instanceTypes = append(instanceTypes, "instance"+strconv.Itoa(i))
+			}
+			nodeClaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: instanceTypes}},
+			}
+			Expect(env.Client.Create(ctx, nodeClaim)).To(Succeed())
+		})
+		It("should error when minValues is greater than the number of unique values specified within In operator", func() {
+			nodeClaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{
+				{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"insance-type-1"}}, MinValues: lo.ToPtr(2)},
+			}
+			Expect(nodeClaim.Validate(ctx)).ToNot(Succeed())
+		})
 	})
 	Context("Kubelet", func() {
 		It("should fail on kubeReserved with invalid keys", func() {