Skip to content

Commit

Permalink
Add topologySpreadConstraints configuration.
Browse files Browse the repository at this point in the history
  • Loading branch information
@laiminhtrung1997
laiminhtrung1997 committed Feb 4, 2024
1 parent 8a1b2f4 commit 68ea861
Show file tree
Hide file tree
Showing 6 changed files with 235 additions and 87 deletions.
71 changes: 71 additions & 0 deletions pkg/apis/acid.zalan.do/v1/crds.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1323,6 +1323,9 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
"enable_sidecars": {
Type: "boolean",
},
"enable_topology_spread_constraints": {
Type: "boolean",
},
"ignored_annotations": {
Type: "array",
Items: &apiextv1.JSONSchemaPropsOrArray{
Expand Down Expand Up @@ -1531,6 +1534,74 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
},
},
},
"topology_spread_constraints": {
Type: "array",
Nullable: true,
Items: &apiextv1.JSONSchemaPropsOrArray{
Schema: &apiextv1.JSONSchemaProps{
Type: "object",
Required: []string{"max_skew", "topology_key", "when_unsatisfiable"},
Properties: map[string]apiextv1.JSONSchemaProps{
"max_skew": {
Type: "integer",
Format: "int32",
},
"topology_key": {
Type: "string",
},
"when_unsatisfiable": {
Type: "string",
Enum: []apiextv1.JSON{
{
Raw: []byte(`"DoNotSchedule"`),
},
{
Raw: []byte(`"ScheduleAnyway"`),
},
},
},
"min_domains": {
Type: "integer",
Nullable: true,
Format: "int32",
},
"node_affinity_policy": {
Type: "string",
Nullable: true,
Enum: []apiextv1.JSON{
{
Raw: []byte(`"Ignore"`),
},
{
Raw: []byte(`"Honor"`),
},
},
},
"node_taints_policy": {
Type: "string",
Nullable: true,
Enum: []apiextv1.JSON{
{
Raw: []byte(`"Ignore"`),
},
{
Raw: []byte(`"Honor"`),
},
},
},
"match_label_keys": {
Type: "array",
Nullable: true,
Items: &apiextv1.JSONSchemaPropsOrArray{
Schema: &apiextv1.JSONSchemaProps{
Type: "string",
},
},
},
},
},
},
},
"watched_namespace": {
Type: "string",
},
Expand Down
28 changes: 15 additions & 13 deletions pkg/apis/acid.zalan.do/v1/operator_configuration_type.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,19 +92,21 @@ type KubernetesMetaConfiguration struct {
NodeReadinessLabelMerge string `json:"node_readiness_label_merge,omitempty"`
CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"`
// TODO: use a proper toleration structure?
PodToleration map[string]string `json:"toleration,omitempty"`
PodEnvironmentConfigMap spec.NamespacedName `json:"pod_environment_configmap,omitempty"`
PodEnvironmentSecret string `json:"pod_environment_secret,omitempty"`
PodPriorityClassName string `json:"pod_priority_class_name,omitempty"`
MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"`
EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"`
PodAntiAffinityPreferredDuringScheduling bool `json:"pod_antiaffinity_preferred_during_scheduling,omitempty"`
PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"`
PodManagementPolicy string `json:"pod_management_policy,omitempty"`
PersistentVolumeClaimRetentionPolicy map[string]string `json:"persistent_volume_claim_retention_policy,omitempty"`
EnableReadinessProbe bool `json:"enable_readiness_probe,omitempty"`
EnableCrossNamespaceSecret bool `json:"enable_cross_namespace_secret,omitempty"`
EnableFinalizers *bool `json:"enable_finalizers,omitempty"`
PodToleration map[string]string `json:"toleration,omitempty"`
PodEnvironmentConfigMap spec.NamespacedName `json:"pod_environment_configmap,omitempty"`
PodEnvironmentSecret string `json:"pod_environment_secret,omitempty"`
PodPriorityClassName string `json:"pod_priority_class_name,omitempty"`
MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"`
EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"`
PodAntiAffinityPreferredDuringScheduling bool `json:"pod_antiaffinity_preferred_during_scheduling,omitempty"`
PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"`
PodManagementPolicy string `json:"pod_management_policy,omitempty"`
PersistentVolumeClaimRetentionPolicy map[string]string `json:"persistent_volume_claim_retention_policy,omitempty"`
EnableReadinessProbe bool `json:"enable_readiness_probe,omitempty"`
EnableCrossNamespaceSecret bool `json:"enable_cross_namespace_secret,omitempty"`
EnableFinalizers *bool `json:"enable_finalizers,omitempty"`
EnableTopologySpreadConstraints bool `json:"enable_topology_spread_constraints,omitempty"`
TopologySpreadConstraints []*config.TopologySpreadConstraint `json:"topology_spread_constraints,omitempty"`
}

// PostgresPodResourcesDefaults defines the spec of default resources
Expand Down
5 changes: 5 additions & 0 deletions pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

44 changes: 42 additions & 2 deletions pkg/cluster/k8sres.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -578,6 +578,36 @@ func generatePodAntiAffinity(podAffinityTerm v1.PodAffinityTerm, preferredDuring
return podAntiAffinity
}

func generateTopologySpreadConstraints(labels labels.Set, topologySpreadConstraintObjs []*config.TopologySpreadConstraint) []v1.TopologySpreadConstraint {
var topologySpreadConstraints []v1.TopologySpreadConstraint
var nodeAffinityPolicy *v1.NodeInclusionPolicy
var nodeTaintsPolicy *v1.NodeInclusionPolicy
for _, topologySpreadConstraintObj := range topologySpreadConstraintObjs {
if topologySpreadConstraintObj.NodeAffinityPolicy != nil {
nodeAffinityPolicy = (*v1.NodeInclusionPolicy)(topologySpreadConstraintObj.NodeAffinityPolicy)
}
if topologySpreadConstraintObj.NodeTaintsPolicy != nil {
nodeTaintsPolicy = (*v1.NodeInclusionPolicy)(topologySpreadConstraintObj.NodeTaintsPolicy)
}
topologySpreadConstraint := v1.TopologySpreadConstraint{
MaxSkew: topologySpreadConstraintObj.MaxSkew,
TopologyKey: topologySpreadConstraintObj.TopologyKey,
WhenUnsatisfiable: v1.UnsatisfiableConstraintAction(topologySpreadConstraintObj.WhenUnsatisfiable),
LabelSelector: &metav1.LabelSelector{
MatchLabels: labels,
},
MinDomains: topologySpreadConstraintObj.MinDomains,
NodeAffinityPolicy: nodeAffinityPolicy,
NodeTaintsPolicy: nodeTaintsPolicy,
MatchLabelKeys: topologySpreadConstraintObj.MatchLabelKeys,
}
topologySpreadConstraints = append(topologySpreadConstraints, topologySpreadConstraint)
nodeAffinityPolicy = nil
nodeTaintsPolicy = nil
}
return topologySpreadConstraints
}

func tolerations(tolerationsSpec *[]v1.Toleration, podToleration map[string]string) []v1.Toleration {
// allow to override tolerations by postgresql manifest
if len(*tolerationsSpec) > 0 {
Expand Down Expand Up @@ -794,6 +824,8 @@ func (c *Cluster) generatePodTemplate(
additionalSecretMount string,
additionalSecretMountPath string,
additionalVolumes []acidv1.AdditionalVolume,
topologySpreadConstraint bool,
topologySpreadConstraintsArr []*config.TopologySpreadConstraint,
) (*v1.PodTemplateSpec, error) {

terminateGracePeriodSeconds := terminateGracePeriod
Expand Down Expand Up @@ -842,6 +874,10 @@ func (c *Cluster) generatePodTemplate(
podSpec.Affinity = nodeAffinity
}

if topologySpreadConstraint && len(topologySpreadConstraintsArr) > 0 {
podSpec.TopologySpreadConstraints = generateTopologySpreadConstraints(labels, topologySpreadConstraintsArr)
}

if priorityClassName != "" {
podSpec.PriorityClassName = priorityClassName
}
Expand Down Expand Up @@ -1449,7 +1485,9 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef
c.OpConfig.PodAntiAffinityPreferredDuringScheduling,
c.OpConfig.AdditionalSecretMount,
c.OpConfig.AdditionalSecretMountPath,
additionalVolumes)
additionalVolumes,
c.OpConfig.EnableTopologySpreadConstraints,
c.OpConfig.TopologySpreadConstraints)

if err != nil {
return nil, fmt.Errorf("could not generate pod template: %v", err)
Expand Down Expand Up @@ -2284,7 +2322,9 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1.CronJob, error) {
false,
c.OpConfig.AdditionalSecretMount,
c.OpConfig.AdditionalSecretMountPath,
[]acidv1.AdditionalVolume{}); err != nil {
[]acidv1.AdditionalVolume{},
false,
[]*config.TopologySpreadConstraint{}); err != nil {
return nil, fmt.Errorf("could not generate pod template for logical backup pod: %v", err)
}

Expand Down
18 changes: 18 additions & 0 deletions pkg/controller/operator_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,5 +285,23 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
fromCRD.ConnectionPooler.MaxDBConnections,
k8sutil.Int32ToPointer(constants.ConnectionPoolerMaxDBConnections))

result.EnableTopologySpreadConstraints = fromCRD.Kubernetes.EnableTopologySpreadConstraints
if fromCRD.Kubernetes.TopologySpreadConstraints != nil {
result.TopologySpreadConstraints = []*config.TopologySpreadConstraint{}
for _, topologySpreadConstraint := range fromCRD.Kubernetes.TopologySpreadConstraints {
result.TopologySpreadConstraints = append(
result.TopologySpreadConstraints,
&config.TopologySpreadConstraint{
MaxSkew: topologySpreadConstraint.MaxSkew,
TopologyKey: topologySpreadConstraint.TopologyKey,
WhenUnsatisfiable: topologySpreadConstraint.WhenUnsatisfiable,
MinDomains: topologySpreadConstraint.MinDomains,
NodeAffinityPolicy: topologySpreadConstraint.NodeAffinityPolicy,
NodeTaintsPolicy: topologySpreadConstraint.NodeTaintsPolicy,
MatchLabelKeys: topologySpreadConstraint.MatchLabelKeys,
})
}
}

return result
}
Loading

0 comments on commit 68ea861

Please sign in to comment.