From c238a0d641dc9146064430e1566bfad947f272a0 Mon Sep 17 00:00:00 2001 From: Kirby Chin Date: Thu, 2 Nov 2023 11:26:05 -0400 Subject: [PATCH] Add WebSphereLibertyApplication and Semeru toplogy spread constraints (#576) * Add wlapp/semeru toplogy spread constraints * Get semeru topologySpreadConstraints from wlapp * Update comment --- api/v1/webspherelibertyapplication_types.go | 33 ++++ api/v1/zz_generated.deepcopy.go | 36 ++++ ...bsphere-liberty.clusterserviceversion.yaml | 14 +- ....ibm.com_webspherelibertyapplications.yaml | 183 ++++++++++++++++++ ....ibm.com_webspherelibertyapplications.yaml | 183 ++++++++++++++++++ ...bsphere-liberty.clusterserviceversion.yaml | 12 ++ controllers/semeru_compiler.go | 22 ++- go.mod | 2 +- go.sum | 4 +- .../kubectl/websphereliberty-app-crd.yaml | 183 ++++++++++++++++++ .../daily/base/websphere-liberty-crd.yaml | 183 ++++++++++++++++++ 11 files changed, 845 insertions(+), 10 deletions(-) diff --git a/api/v1/webspherelibertyapplication_types.go b/api/v1/webspherelibertyapplication_types.go index db599e38..7c7463df 100644 --- a/api/v1/webspherelibertyapplication_types.go +++ b/api/v1/webspherelibertyapplication_types.go @@ -163,6 +163,9 @@ type WebSphereLibertyApplicationSpec struct { // Security context for the application container. // +operator-sdk:csv:customresourcedefinitions:order=31,type=spec,displayName="Security Context" SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` + + // +operator-sdk:csv:customresourcedefinitions:order=26,type=spec,displayName="Topology Spread Constraints" + TopologySpreadConstraints *WebSphereLibertyApplicationTopologySpreadConstraints `json:"topologySpreadConstraints,omitempty"` } // License information is required. @@ -225,6 +228,17 @@ const ( LicenseEntitlementWSHE LicenseEntitlement = "IBM WebSphere Hybrid Edition" ) +// Defines the topology spread constraints +type WebSphereLibertyApplicationTopologySpreadConstraints struct { + // The list of TopologySpreadConstraints for the application instance and if applicable, the Semeru Cloud Compiler instance. + // +operator-sdk:csv:customresourcedefinitions:order=1,type=spec,displayName="Constraints" + Constraints *[]corev1.TopologySpreadConstraint `json:"constraints,omitempty"` + + // Whether the operator should disable its default set of TopologySpreadConstraints. Defaults to false. + // +operator-sdk:csv:customresourcedefinitions:order=1,type=spec,displayName="Disable Operator Defaults",xDescriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch" + DisableOperatorDefaults *bool `json:"disableOperatorDefaults,omitempty"` +} + // Defines the service account type WebSphereLibertyApplicationServiceAccount struct { // Whether the Service Account token should be mounted into the application pods. Defaults to true. @@ -1213,6 +1227,25 @@ func (scc *WebSphereLibertyApplicationSemeruCloudCompiler) GetReplicas() *int32 return &one } +// GetTopologySpreadConstraints returns the pod topology spread constraints configuration +func (cr *WebSphereLibertyApplication) GetTopologySpreadConstraints() common.BaseComponentTopologySpreadConstraints { + if cr.Spec.TopologySpreadConstraints == nil { + return nil + } + return cr.Spec.TopologySpreadConstraints +} + +func (cr *WebSphereLibertyApplicationTopologySpreadConstraints) GetConstraints() *[]corev1.TopologySpreadConstraint { + if cr.Constraints == nil { + return nil + } + return cr.Constraints +} + +func (cr *WebSphereLibertyApplicationTopologySpreadConstraints) GetDisableOperatorDefaults() *bool { + return cr.DisableOperatorDefaults +} + // Initialize sets default values func (cr *WebSphereLibertyApplication) Initialize() { if cr.Spec.PullPolicy == nil { diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 98901ea6..123ea00f 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -891,6 +891,11 @@ func (in *WebSphereLibertyApplicationSpec) DeepCopyInto(out *WebSphereLibertyApp *out = new(corev1.SecurityContext) (*in).DeepCopyInto(*out) } + if in.TopologySpreadConstraints != nil { + in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints + *out = new(WebSphereLibertyApplicationTopologySpreadConstraints) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebSphereLibertyApplicationSpec. @@ -1005,6 +1010,37 @@ func (in *WebSphereLibertyApplicationStorage) DeepCopy() *WebSphereLibertyApplic return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WebSphereLibertyApplicationTopologySpreadConstraints) DeepCopyInto(out *WebSphereLibertyApplicationTopologySpreadConstraints) { + *out = *in + if in.Constraints != nil { + in, out := &in.Constraints, &out.Constraints + *out = new([]corev1.TopologySpreadConstraint) + if **in != nil { + in, out := *in, *out + *out = make([]corev1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + } + if in.DisableOperatorDefaults != nil { + in, out := &in.DisableOperatorDefaults, &out.DisableOperatorDefaults + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebSphereLibertyApplicationTopologySpreadConstraints. +func (in *WebSphereLibertyApplicationTopologySpreadConstraints) DeepCopy() *WebSphereLibertyApplicationTopologySpreadConstraints { + if in == nil { + return nil + } + out := new(WebSphereLibertyApplicationTopologySpreadConstraints) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WebSphereLibertyDump) DeepCopyInto(out *WebSphereLibertyDump) { *out = *in diff --git a/bundle/manifests/ibm-websphere-liberty.clusterserviceversion.yaml b/bundle/manifests/ibm-websphere-liberty.clusterserviceversion.yaml index 9e5a74f1..5d7595cc 100644 --- a/bundle/manifests/ibm-websphere-liberty.clusterserviceversion.yaml +++ b/bundle/manifests/ibm-websphere-liberty.clusterserviceversion.yaml @@ -62,7 +62,7 @@ metadata: capabilities: Auto Pilot categories: Application Runtime containerImage: icr.io/cpopen/websphere-liberty-operator:daily - createdAt: "2023-10-30T04:18:07Z" + createdAt: "2023-11-02T15:00:01Z" description: Deploy and manage containerized Liberty applications olm.skipRange: '>=1.0.0 <1.3.0' operators.openshift.io/infrastructure-features: '["disconnected"]' @@ -140,6 +140,16 @@ spec: - description: The unique ID for the provider. Default value is oidc. displayName: ID path: sso.oidc[0].id + - description: The list of TopologySpreadConstraints for the application instance + and if applicable, the Semeru Cloud Compiler instance. + displayName: Constraints + path: topologySpreadConstraints.constraints + - description: Whether the operator should disable its default set of TopologySpreadConstraints. + Defaults to false. + displayName: Disable Operator Defaults + path: topologySpreadConstraints.disableOperatorDefaults + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - description: Name of the application. Defaults to the name of this custom resource. displayName: Application Name @@ -379,6 +389,8 @@ spec: path: statefulSet.storage.className x-descriptors: - urn:alm:descriptor:com.tectonic.ui:text + - displayName: Topology Spread Constraints + path: topologySpreadConstraints - description: Represents a volume with data that is accessible to the application container. displayName: Volumes diff --git a/bundle/manifests/liberty.websphere.ibm.com_webspherelibertyapplications.yaml b/bundle/manifests/liberty.websphere.ibm.com_webspherelibertyapplications.yaml index 24eb407c..d5596fcf 100644 --- a/bundle/manifests/liberty.websphere.ibm.com_webspherelibertyapplications.yaml +++ b/bundle/manifests/liberty.websphere.ibm.com_webspherelibertyapplications.yaml @@ -5673,6 +5673,189 @@ spec: type: string type: object type: object + topologySpreadConstraints: + description: Defines the topology spread constraints + properties: + constraints: + description: The list of TopologySpreadConstraints for the application + instance and if applicable, the Semeru Cloud Compiler instance. + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod label keys to + select the pods over which spreading will be calculated. + The keys are used to lookup values from the incoming pod + labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading + will be calculated for the incoming pod. Keys that don't + exist in the incoming pod labels will be ignored. A null + or empty list means only match against labelSelector. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: 'MaxSkew describes the degree to which pods + may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global + minimum. The global minimum is the minimum number of matching + pods in an eligible domain or zero if the number of eligible + domains is less than MinDomains. For example, in a 3-zone + cluster, MaxSkew is set to 1, and pods with the same labelSelector + spread as 2/2/1: In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | | P P | P P | P | - + if MaxSkew is 1, incoming pod can only be scheduled to + zone3 to become 2/2/2; scheduling it onto zone1(zone2) + would make the ActualSkew(3-1) on zone1(zone2) violate + MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled + onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that + satisfy it. It''s a required field. Default value is 1 + and 0 is not allowed.' + format: int32 + type: integer + minDomains: + description: "MinDomains indicates a minimum number of eligible + domains. When the number of eligible domains with matching + topology keys is less than minDomains, Pod Topology Spread + treats \"global minimum\" as 0, and then the calculation + of Skew is performed. And when the number of eligible + domains with matching topology keys equals or greater + than minDomains, this value has no effect on scheduling. + As a result, when the number of eligible domains is less + than minDomains, scheduler won't schedule more than maxSkew + Pods to those domains. If value is nil, the constraint + behaves as if MinDomains is equal to 1. Valid values are + integers greater than 0. When value is not nil, WhenUnsatisfiable + must be DoNotSchedule. \n For example, in a 3-zone cluster, + MaxSkew is set to 2, MinDomains is set to 5 and pods with + the same labelSelector spread as 2/2/2: | zone1 | zone2 + | zone3 | | P P | P P | P P | The number of domains + is less than 5(MinDomains), so \"global minimum\" is treated + as 0. In this situation, new pod with the same labelSelector + cannot be scheduled, because computed skew will be 3(3 + - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. \n This is a beta field and requires + the MinDomainsInPodTopologySpread feature gate to be enabled + (enabled by default)." + format: int32 + type: integer + nodeAffinityPolicy: + description: "NodeAffinityPolicy indicates how we will treat + Pod's nodeAffinity/nodeSelector when calculating pod topology + spread skew. Options are: - Honor: only nodes matching + nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes + are included in the calculations. \n If this value is + nil, the behavior is equivalent to the Honor policy. This + is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + nodeTaintsPolicy: + description: "NodeTaintsPolicy indicates how we will treat + node taints when calculating pod topology spread skew. + Options are: - Honor: nodes without taints, along with + tainted nodes for which the incoming pod has a toleration, + are included. - Ignore: node taints are ignored. All nodes + are included. \n If this value is nil, the behavior is + equivalent to the Ignore policy. This is a alpha-level + feature enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + topologyKey: + description: TopologyKey is the key of node labels. Nodes + that have a label with this key and identical values are + considered to be in the same topology. We consider each + as a "bucket", and try to put balanced number + of pods into each bucket. We define a domain as a particular + instance of a topology. Also, we define an eligible domain + as a domain whose nodes meet the requirements of nodeAffinityPolicy + and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", + each Node is a domain of that topology. And, if TopologyKey + is "topology.kubernetes.io/zone", each zone is a domain + of that topology. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal with + a pod if it doesn''t satisfy the spread constraint. - + DoNotSchedule (default) tells the scheduler not to schedule + it. - ScheduleAnyway tells the scheduler to schedule the + pod in any location, but giving higher precedence to topologies + that would help reduce the skew. A constraint is considered + "Unsatisfiable" for an incoming pod if and only if every + possible node assignment for that pod would violate "MaxSkew" + on some topology. For example, in a 3-zone cluster, MaxSkew + is set to 1, and pods with the same labelSelector spread + as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming + pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) + as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). + In other words, the cluster can still be imbalanced, but + scheduler won''t make it *more* imbalanced. It''s a required + field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + disableOperatorDefaults: + description: Whether the operator should disable its default set + of TopologySpreadConstraints. Defaults to false. + type: boolean + type: object volumeMounts: description: Represents where to mount the volumes into the application container. diff --git a/config/crd/bases/liberty.websphere.ibm.com_webspherelibertyapplications.yaml b/config/crd/bases/liberty.websphere.ibm.com_webspherelibertyapplications.yaml index dc801e3e..dfaa5c13 100644 --- a/config/crd/bases/liberty.websphere.ibm.com_webspherelibertyapplications.yaml +++ b/config/crd/bases/liberty.websphere.ibm.com_webspherelibertyapplications.yaml @@ -5670,6 +5670,189 @@ spec: type: string type: object type: object + topologySpreadConstraints: + description: Defines the topology spread constraints + properties: + constraints: + description: The list of TopologySpreadConstraints for the application + instance and if applicable, the Semeru Cloud Compiler instance. + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod label keys to + select the pods over which spreading will be calculated. + The keys are used to lookup values from the incoming pod + labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading + will be calculated for the incoming pod. Keys that don't + exist in the incoming pod labels will be ignored. A null + or empty list means only match against labelSelector. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: 'MaxSkew describes the degree to which pods + may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global + minimum. The global minimum is the minimum number of matching + pods in an eligible domain or zero if the number of eligible + domains is less than MinDomains. For example, in a 3-zone + cluster, MaxSkew is set to 1, and pods with the same labelSelector + spread as 2/2/1: In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | | P P | P P | P | - + if MaxSkew is 1, incoming pod can only be scheduled to + zone3 to become 2/2/2; scheduling it onto zone1(zone2) + would make the ActualSkew(3-1) on zone1(zone2) violate + MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled + onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that + satisfy it. It''s a required field. Default value is 1 + and 0 is not allowed.' + format: int32 + type: integer + minDomains: + description: "MinDomains indicates a minimum number of eligible + domains. When the number of eligible domains with matching + topology keys is less than minDomains, Pod Topology Spread + treats \"global minimum\" as 0, and then the calculation + of Skew is performed. And when the number of eligible + domains with matching topology keys equals or greater + than minDomains, this value has no effect on scheduling. + As a result, when the number of eligible domains is less + than minDomains, scheduler won't schedule more than maxSkew + Pods to those domains. If value is nil, the constraint + behaves as if MinDomains is equal to 1. Valid values are + integers greater than 0. When value is not nil, WhenUnsatisfiable + must be DoNotSchedule. \n For example, in a 3-zone cluster, + MaxSkew is set to 2, MinDomains is set to 5 and pods with + the same labelSelector spread as 2/2/2: | zone1 | zone2 + | zone3 | | P P | P P | P P | The number of domains + is less than 5(MinDomains), so \"global minimum\" is treated + as 0. In this situation, new pod with the same labelSelector + cannot be scheduled, because computed skew will be 3(3 + - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. \n This is a beta field and requires + the MinDomainsInPodTopologySpread feature gate to be enabled + (enabled by default)." + format: int32 + type: integer + nodeAffinityPolicy: + description: "NodeAffinityPolicy indicates how we will treat + Pod's nodeAffinity/nodeSelector when calculating pod topology + spread skew. Options are: - Honor: only nodes matching + nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes + are included in the calculations. \n If this value is + nil, the behavior is equivalent to the Honor policy. This + is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + nodeTaintsPolicy: + description: "NodeTaintsPolicy indicates how we will treat + node taints when calculating pod topology spread skew. + Options are: - Honor: nodes without taints, along with + tainted nodes for which the incoming pod has a toleration, + are included. - Ignore: node taints are ignored. All nodes + are included. \n If this value is nil, the behavior is + equivalent to the Ignore policy. This is a alpha-level + feature enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + topologyKey: + description: TopologyKey is the key of node labels. Nodes + that have a label with this key and identical values are + considered to be in the same topology. We consider each + as a "bucket", and try to put balanced number + of pods into each bucket. We define a domain as a particular + instance of a topology. Also, we define an eligible domain + as a domain whose nodes meet the requirements of nodeAffinityPolicy + and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", + each Node is a domain of that topology. And, if TopologyKey + is "topology.kubernetes.io/zone", each zone is a domain + of that topology. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal with + a pod if it doesn''t satisfy the spread constraint. - + DoNotSchedule (default) tells the scheduler not to schedule + it. - ScheduleAnyway tells the scheduler to schedule the + pod in any location, but giving higher precedence to topologies + that would help reduce the skew. A constraint is considered + "Unsatisfiable" for an incoming pod if and only if every + possible node assignment for that pod would violate "MaxSkew" + on some topology. For example, in a 3-zone cluster, MaxSkew + is set to 1, and pods with the same labelSelector spread + as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming + pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) + as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). + In other words, the cluster can still be imbalanced, but + scheduler won''t make it *more* imbalanced. It''s a required + field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + disableOperatorDefaults: + description: Whether the operator should disable its default set + of TopologySpreadConstraints. Defaults to false. + type: boolean + type: object volumeMounts: description: Represents where to mount the volumes into the application container. diff --git a/config/manifests/bases/ibm-websphere-liberty.clusterserviceversion.yaml b/config/manifests/bases/ibm-websphere-liberty.clusterserviceversion.yaml index 4e746e63..3c474746 100644 --- a/config/manifests/bases/ibm-websphere-liberty.clusterserviceversion.yaml +++ b/config/manifests/bases/ibm-websphere-liberty.clusterserviceversion.yaml @@ -81,6 +81,16 @@ spec: - description: The unique ID for the provider. Default value is oidc. displayName: ID path: sso.oidc[0].id + - description: The list of TopologySpreadConstraints for the application instance + and if applicable, the Semeru Cloud Compiler instance. + displayName: Constraints + path: topologySpreadConstraints.constraints + - description: Whether the operator should disable its default set of TopologySpreadConstraints. + Defaults to false. + displayName: Disable Operator Defaults + path: topologySpreadConstraints.disableOperatorDefaults + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - description: Name of the application. Defaults to the name of this custom resource. displayName: Application Name @@ -320,6 +330,8 @@ spec: path: statefulSet.storage.className x-descriptors: - urn:alm:descriptor:com.tectonic.ui:text + - displayName: Topology Spread Constraints + path: topologySpreadConstraints - description: Represents a volume with data that is accessible to the application container. displayName: Volumes diff --git a/controllers/semeru_compiler.go b/controllers/semeru_compiler.go index 000f4575..b88cab62 100644 --- a/controllers/semeru_compiler.go +++ b/controllers/semeru_compiler.go @@ -298,6 +298,9 @@ func (r *ReconcileWebSphereLiberty) reconcileSemeruDeployment(wlva *wlv1.WebSphe PeriodSeconds: 5, } + semeruPodMatchLabels := map[string]string{ + "app.kubernetes.io/instance": getSemeruCompilerNameWithGeneration(wlva), + } deploy.Spec.Template = corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: getLabels(wlva), @@ -312,9 +315,7 @@ func (r *ReconcileWebSphereLiberty) reconcileSemeruDeployment(wlva *wlv1.WebSphe PodAffinityTerm: corev1.PodAffinityTerm{ TopologyKey: "topology.kubernetes.io/zone", LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app.kubernetes.io/instance": getSemeruCompilerNameWithGeneration(wlva), - }, + MatchLabels: semeruPodMatchLabels, }, }, }, @@ -323,9 +324,7 @@ func (r *ReconcileWebSphereLiberty) reconcileSemeruDeployment(wlva *wlv1.WebSphe PodAffinityTerm: corev1.PodAffinityTerm{ TopologyKey: "kubernetes.io/hostname", LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app.kubernetes.io/instance": getSemeruCompilerNameWithGeneration(wlva), - }, + MatchLabels: semeruPodMatchLabels, }, }, }, @@ -382,6 +381,17 @@ func (r *ReconcileWebSphereLiberty) reconcileSemeruDeployment(wlva *wlv1.WebSphe }, } + // Configure TopologySpreadConstraints from the WebSphereLibertyApplication CR + deploy.Spec.Template.Spec.TopologySpreadConstraints = make([]corev1.TopologySpreadConstraint, 0) + topologySpreadConstraintsConfig := wlva.GetTopologySpreadConstraints() + if topologySpreadConstraintsConfig == nil || topologySpreadConstraintsConfig.GetDisableOperatorDefaults() == nil || !*topologySpreadConstraintsConfig.GetDisableOperatorDefaults() { + utils.CustomizeTopologySpreadConstraints(&deploy.Spec.Template, semeruPodMatchLabels) + } + if topologySpreadConstraintsConfig != nil && topologySpreadConstraintsConfig.GetConstraints() != nil { + deploy.Spec.Template.Spec.TopologySpreadConstraints = utils.MergeTopologySpreadConstraints(deploy.Spec.Template.Spec.TopologySpreadConstraints, + *topologySpreadConstraintsConfig.GetConstraints()) + } + // Copy the service account from the WebSphereLibertyApplcation CR if saName := utils.GetServiceAccountName(wlva); saName != "" { deploy.Spec.Template.Spec.ServiceAccountName = saName diff --git a/go.mod b/go.mod index 1f96d04d..31b701fc 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/WASdev/websphere-liberty-operator go 1.21 require ( - github.com/application-stacks/runtime-component-operator v1.0.0-20220602-0850.0.20231030173230-96eb9e69774c + github.com/application-stacks/runtime-component-operator v1.0.0-20220602-0850.0.20231101204909-42a1eb2ab28b github.com/cert-manager/cert-manager v1.10.2 github.com/go-logr/logr v1.2.4 github.com/openshift/api v0.0.0-20230928134114-673ed0cfc7f1 diff --git a/go.sum b/go.sum index 89bd563f..c1771400 100644 --- a/go.sum +++ b/go.sum @@ -54,8 +54,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/application-stacks/runtime-component-operator v1.0.0-20220602-0850.0.20231030173230-96eb9e69774c h1:47PqbNZUUYKccRv+1dW8kcdLx7muWvTa4i5BTOc52rQ= -github.com/application-stacks/runtime-component-operator v1.0.0-20220602-0850.0.20231030173230-96eb9e69774c/go.mod h1:Yhxg+reeWEw4OnPUsX3ZYogS3RbtmC0M20TzX/w4N0I= +github.com/application-stacks/runtime-component-operator v1.0.0-20220602-0850.0.20231101204909-42a1eb2ab28b h1:Eb+7GZz2VcC83mnMgDEtCJe8cQF2fM58rrG+TwnGj3Y= +github.com/application-stacks/runtime-component-operator v1.0.0-20220602-0850.0.20231101204909-42a1eb2ab28b/go.mod h1:Yhxg+reeWEw4OnPUsX3ZYogS3RbtmC0M20TzX/w4N0I= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= diff --git a/internal/deploy/kubectl/websphereliberty-app-crd.yaml b/internal/deploy/kubectl/websphereliberty-app-crd.yaml index 586d8285..f1c9ef9d 100644 --- a/internal/deploy/kubectl/websphereliberty-app-crd.yaml +++ b/internal/deploy/kubectl/websphereliberty-app-crd.yaml @@ -5672,6 +5672,189 @@ spec: type: string type: object type: object + topologySpreadConstraints: + description: Defines the topology spread constraints + properties: + constraints: + description: The list of TopologySpreadConstraints for the application + instance and if applicable, the Semeru Cloud Compiler instance. + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod label keys to + select the pods over which spreading will be calculated. + The keys are used to lookup values from the incoming pod + labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading + will be calculated for the incoming pod. Keys that don't + exist in the incoming pod labels will be ignored. A null + or empty list means only match against labelSelector. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: 'MaxSkew describes the degree to which pods + may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global + minimum. The global minimum is the minimum number of matching + pods in an eligible domain or zero if the number of eligible + domains is less than MinDomains. For example, in a 3-zone + cluster, MaxSkew is set to 1, and pods with the same labelSelector + spread as 2/2/1: In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | | P P | P P | P | - + if MaxSkew is 1, incoming pod can only be scheduled to + zone3 to become 2/2/2; scheduling it onto zone1(zone2) + would make the ActualSkew(3-1) on zone1(zone2) violate + MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled + onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that + satisfy it. It''s a required field. Default value is 1 + and 0 is not allowed.' + format: int32 + type: integer + minDomains: + description: "MinDomains indicates a minimum number of eligible + domains. When the number of eligible domains with matching + topology keys is less than minDomains, Pod Topology Spread + treats \"global minimum\" as 0, and then the calculation + of Skew is performed. And when the number of eligible + domains with matching topology keys equals or greater + than minDomains, this value has no effect on scheduling. + As a result, when the number of eligible domains is less + than minDomains, scheduler won't schedule more than maxSkew + Pods to those domains. If value is nil, the constraint + behaves as if MinDomains is equal to 1. Valid values are + integers greater than 0. When value is not nil, WhenUnsatisfiable + must be DoNotSchedule. \n For example, in a 3-zone cluster, + MaxSkew is set to 2, MinDomains is set to 5 and pods with + the same labelSelector spread as 2/2/2: | zone1 | zone2 + | zone3 | | P P | P P | P P | The number of domains + is less than 5(MinDomains), so \"global minimum\" is treated + as 0. In this situation, new pod with the same labelSelector + cannot be scheduled, because computed skew will be 3(3 + - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. \n This is a beta field and requires + the MinDomainsInPodTopologySpread feature gate to be enabled + (enabled by default)." + format: int32 + type: integer + nodeAffinityPolicy: + description: "NodeAffinityPolicy indicates how we will treat + Pod's nodeAffinity/nodeSelector when calculating pod topology + spread skew. Options are: - Honor: only nodes matching + nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes + are included in the calculations. \n If this value is + nil, the behavior is equivalent to the Honor policy. This + is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + nodeTaintsPolicy: + description: "NodeTaintsPolicy indicates how we will treat + node taints when calculating pod topology spread skew. + Options are: - Honor: nodes without taints, along with + tainted nodes for which the incoming pod has a toleration, + are included. - Ignore: node taints are ignored. All nodes + are included. \n If this value is nil, the behavior is + equivalent to the Ignore policy. This is a alpha-level + feature enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + topologyKey: + description: TopologyKey is the key of node labels. Nodes + that have a label with this key and identical values are + considered to be in the same topology. We consider each + as a "bucket", and try to put balanced number + of pods into each bucket. We define a domain as a particular + instance of a topology. Also, we define an eligible domain + as a domain whose nodes meet the requirements of nodeAffinityPolicy + and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", + each Node is a domain of that topology. And, if TopologyKey + is "topology.kubernetes.io/zone", each zone is a domain + of that topology. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal with + a pod if it doesn''t satisfy the spread constraint. - + DoNotSchedule (default) tells the scheduler not to schedule + it. - ScheduleAnyway tells the scheduler to schedule the + pod in any location, but giving higher precedence to topologies + that would help reduce the skew. A constraint is considered + "Unsatisfiable" for an incoming pod if and only if every + possible node assignment for that pod would violate "MaxSkew" + on some topology. For example, in a 3-zone cluster, MaxSkew + is set to 1, and pods with the same labelSelector spread + as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming + pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) + as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). + In other words, the cluster can still be imbalanced, but + scheduler won''t make it *more* imbalanced. It''s a required + field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + disableOperatorDefaults: + description: Whether the operator should disable its default set + of TopologySpreadConstraints. Defaults to false. + type: boolean + type: object volumeMounts: description: Represents where to mount the volumes into the application container. diff --git a/internal/deploy/kustomize/daily/base/websphere-liberty-crd.yaml b/internal/deploy/kustomize/daily/base/websphere-liberty-crd.yaml index 586d8285..f1c9ef9d 100644 --- a/internal/deploy/kustomize/daily/base/websphere-liberty-crd.yaml +++ b/internal/deploy/kustomize/daily/base/websphere-liberty-crd.yaml @@ -5672,6 +5672,189 @@ spec: type: string type: object type: object + topologySpreadConstraints: + description: Defines the topology spread constraints + properties: + constraints: + description: The list of TopologySpreadConstraints for the application + instance and if applicable, the Semeru Cloud Compiler instance. + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod label keys to + select the pods over which spreading will be calculated. + The keys are used to lookup values from the incoming pod + labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading + will be calculated for the incoming pod. Keys that don't + exist in the incoming pod labels will be ignored. A null + or empty list means only match against labelSelector. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: 'MaxSkew describes the degree to which pods + may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global + minimum. The global minimum is the minimum number of matching + pods in an eligible domain or zero if the number of eligible + domains is less than MinDomains. For example, in a 3-zone + cluster, MaxSkew is set to 1, and pods with the same labelSelector + spread as 2/2/1: In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | | P P | P P | P | - + if MaxSkew is 1, incoming pod can only be scheduled to + zone3 to become 2/2/2; scheduling it onto zone1(zone2) + would make the ActualSkew(3-1) on zone1(zone2) violate + MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled + onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that + satisfy it. It''s a required field. Default value is 1 + and 0 is not allowed.' + format: int32 + type: integer + minDomains: + description: "MinDomains indicates a minimum number of eligible + domains. When the number of eligible domains with matching + topology keys is less than minDomains, Pod Topology Spread + treats \"global minimum\" as 0, and then the calculation + of Skew is performed. And when the number of eligible + domains with matching topology keys equals or greater + than minDomains, this value has no effect on scheduling. + As a result, when the number of eligible domains is less + than minDomains, scheduler won't schedule more than maxSkew + Pods to those domains. If value is nil, the constraint + behaves as if MinDomains is equal to 1. Valid values are + integers greater than 0. When value is not nil, WhenUnsatisfiable + must be DoNotSchedule. \n For example, in a 3-zone cluster, + MaxSkew is set to 2, MinDomains is set to 5 and pods with + the same labelSelector spread as 2/2/2: | zone1 | zone2 + | zone3 | | P P | P P | P P | The number of domains + is less than 5(MinDomains), so \"global minimum\" is treated + as 0. In this situation, new pod with the same labelSelector + cannot be scheduled, because computed skew will be 3(3 + - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. \n This is a beta field and requires + the MinDomainsInPodTopologySpread feature gate to be enabled + (enabled by default)." + format: int32 + type: integer + nodeAffinityPolicy: + description: "NodeAffinityPolicy indicates how we will treat + Pod's nodeAffinity/nodeSelector when calculating pod topology + spread skew. Options are: - Honor: only nodes matching + nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes + are included in the calculations. \n If this value is + nil, the behavior is equivalent to the Honor policy. This + is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + nodeTaintsPolicy: + description: "NodeTaintsPolicy indicates how we will treat + node taints when calculating pod topology spread skew. + Options are: - Honor: nodes without taints, along with + tainted nodes for which the incoming pod has a toleration, + are included. - Ignore: node taints are ignored. All nodes + are included. \n If this value is nil, the behavior is + equivalent to the Ignore policy. This is a alpha-level + feature enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + topologyKey: + description: TopologyKey is the key of node labels. Nodes + that have a label with this key and identical values are + considered to be in the same topology. We consider each + as a "bucket", and try to put balanced number + of pods into each bucket. We define a domain as a particular + instance of a topology. Also, we define an eligible domain + as a domain whose nodes meet the requirements of nodeAffinityPolicy + and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", + each Node is a domain of that topology. And, if TopologyKey + is "topology.kubernetes.io/zone", each zone is a domain + of that topology. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal with + a pod if it doesn''t satisfy the spread constraint. - + DoNotSchedule (default) tells the scheduler not to schedule + it. - ScheduleAnyway tells the scheduler to schedule the + pod in any location, but giving higher precedence to topologies + that would help reduce the skew. A constraint is considered + "Unsatisfiable" for an incoming pod if and only if every + possible node assignment for that pod would violate "MaxSkew" + on some topology. For example, in a 3-zone cluster, MaxSkew + is set to 1, and pods with the same labelSelector spread + as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming + pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) + as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). + In other words, the cluster can still be imbalanced, but + scheduler won''t make it *more* imbalanced. It''s a required + field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + disableOperatorDefaults: + description: Whether the operator should disable its default set + of TopologySpreadConstraints. Defaults to false. + type: boolean + type: object volumeMounts: description: Represents where to mount the volumes into the application container.