From aad4733f9f097b249b496ca34ab69782a87d1f9b Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Mon, 12 Feb 2024 22:48:27 +0200 Subject: [PATCH 1/4] balloons: allow balloon match by expressions. Implement assigning containers to balloon instances by container match expression defined in balloon definitions. Match expressions are evaluated for containers which are not explicitly assigned to any balloon by an annotation, before namespace-based assignments. If an expression in a balloon definition matches a container, that container is assigned to a balloon instance of that definition. For instance, given this balloon definition among the configuration ... balloonTypes: - name: specialBalloon matchExpressions: - key: pod/labels/app.kubernetes.io/component operator: Equals values: [ "SpecialComponent" ] ... any container not explicitly annotated to some other balloon will get assigned to instances of specialBalloon, if the pod of the container has the label app.kubernetes.io/component=SpecialComponent Similarly, given this balloon definition among the configuration ... balloonTypes: - name: podBalloon matchExpressions: - key: pod/name operator: In values: [ "pod1", "pod2", "pod3" ] ... all containers of pods named pod1, pod2, and pod3 will be assigned to instances of podBalloon. Signed-off-by: Krisztian Litkey --- .../balloons/policy/balloons-policy.go | 17 +++++-- .../bases/config.nri_balloonspolicies.yaml | 45 +++++++++++++++++++ .../crds/config.nri_balloonspolicies.yaml | 45 +++++++++++++++++++ .../v1alpha1/resmgr/policy/balloons/config.go | 5 +++ .../policy/balloons/zz_generated.deepcopy.go | 8 ++++ 5 files changed, 116 insertions(+), 4 deletions(-) diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index e29246061..cd1eb220b 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -354,24 +354,33 @@ func (p *balloons) balloonDefByName(defName string) *BalloonDef { } func (p *balloons) chooseBalloonDef(c cache.Container) (*BalloonDef, error) { - var blnDef *BalloonDef // Case 1: BalloonDef is defined by annotation. if blnDefName, ok := c.GetEffectiveAnnotation(balloonKey); ok { - blnDef = p.balloonDefByName(blnDefName) + blnDef := p.balloonDefByName(blnDefName) if blnDef == nil { return nil, balloonsError("no balloon for annotation %q", blnDefName) } return blnDef, nil } - // Case 2: BalloonDef is defined by the namespace. for _, blnDef := range p.bpoptions.BalloonDefs { + // Case 2: BalloonDef is defined by a match expression. + for _, expr := range blnDef.MatchExpressions { + log.Debugf("- checking expression %s of balloon %q against container %s...", + expr.String(), blnDef.Name, c.PrettyName()) + if expr.Evaluate(c) { + log.Debugf(" => matches") + return blnDef, nil + } + } + + // Case 3: BalloonDef is defined by the namespace. if namespaceMatches(c.GetNamespace(), blnDef.Namespaces) { return blnDef, nil } } - // Case 3: Fallback to the default balloon. + // Case 4: Fallback to the default balloon. return p.defaultBalloonDef, nil } diff --git a/config/crd/bases/config.nri_balloonspolicies.yaml b/config/crd/bases/config.nri_balloonspolicies.yaml index 2fe4b12e8..10f4a15ff 100644 --- a/config/crd/bases/config.nri_balloonspolicies.yaml +++ b/config/crd/bases/config.nri_balloonspolicies.yaml @@ -83,6 +83,51 @@ spec: CpuClass controls how CPUs of a balloon are (re)configured whenever a balloon is created, inflated or deflated. type: string + matchExpressions: + description: |- + MatchExpressions specifies one or more expressions which are evaluated + to see if a container should be assigned into balloon instances from + this definition. + items: + description: |- + Expression describes some runtime-evaluated condition. An expression + consist of a key, an operator and a set of values. An expressions is + evaluated against an object which implements the Evaluable interface. + Evaluating an expression consists of looking up the value for the key + in the object, then using the operator to check it agains the values + of the expression. The result is a single boolean value. An object is + said to satisfy the evaluated expression if this value is true. An + expression can contain 0, 1 or more values depending on the operator. + properties: + key: + description: Key is the expression key. + type: string + operator: + description: Op is the expression operator. + enum: + - Equals + - NotEqual + - In + - NotIn + - Exists + - NotExist + - AlwaysTrue + - Matches + - MatchesNot + - MatchesAny + - MatchesNone + type: string + values: + description: Values contains the values the key value + is evaluated against. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array maxBalloons: description: |- MaxBalloons is the maximum number of balloon instances that diff --git a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml index 2fe4b12e8..10f4a15ff 100644 --- a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml +++ b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml @@ -83,6 +83,51 @@ spec: CpuClass controls how CPUs of a balloon are (re)configured whenever a balloon is created, inflated or deflated. type: string + matchExpressions: + description: |- + MatchExpressions specifies one or more expressions which are evaluated + to see if a container should be assigned into balloon instances from + this definition. + items: + description: |- + Expression describes some runtime-evaluated condition. An expression + consist of a key, an operator and a set of values. An expressions is + evaluated against an object which implements the Evaluable interface. + Evaluating an expression consists of looking up the value for the key + in the object, then using the operator to check it agains the values + of the expression. The result is a single boolean value. An object is + said to satisfy the evaluated expression if this value is true. An + expression can contain 0, 1 or more values depending on the operator. + properties: + key: + description: Key is the expression key. + type: string + operator: + description: Op is the expression operator. + enum: + - Equals + - NotEqual + - In + - NotIn + - Exists + - NotExist + - AlwaysTrue + - Matches + - MatchesNot + - MatchesAny + - MatchesNone + type: string + values: + description: Values contains the values the key value + is evaluated against. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array maxBalloons: description: |- MaxBalloons is the maximum number of balloon instances that diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go index 42af407ca..d3a8a717d 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go @@ -18,6 +18,7 @@ import ( "strings" policy "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + resmgr "github.com/containers/nri-plugins/pkg/apis/resmgr/v1alpha1" "github.com/containers/nri-plugins/pkg/cpuallocator" ) @@ -124,6 +125,10 @@ type BalloonDef struct { // balloon instances from this definition. This is used by // namespace assign methods. Namespaces []string `json:"namespaces,omitempty"` + // MatchExpressions specifies one or more expressions which are evaluated + // to see if a container should be assigned into balloon instances from + // this definition. + MatchExpressions []resmgr.Expression `json:"matchExpressions,omitempty"` // MaxCpus specifies the maximum number of CPUs exclusively // usable by containers in a balloon. Balloon size will not be // inflated larger than MaxCpus. diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go index 33d9aa844..437c4df27 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go @@ -20,6 +20,7 @@ package balloons import ( "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + v1alpha1 "github.com/containers/nri-plugins/pkg/apis/resmgr/v1alpha1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -30,6 +31,13 @@ func (in *BalloonDef) DeepCopyInto(out *BalloonDef) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.MatchExpressions != nil { + in, out := &in.MatchExpressions, &out.MatchExpressions + *out = make([]v1alpha1.Expression, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.PreferSpreadOnPhysicalCores != nil { in, out := &in.PreferSpreadOnPhysicalCores, &out.PreferSpreadOnPhysicalCores *out = new(bool) From fa62ad8f73aeab4247eee927f26372ce7b2db704 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Mon, 12 Feb 2024 22:59:41 +0200 Subject: [PATCH 2/4] config: add custom validation for balloons. Implement custom validation for balloons. The custom validator validates all container match expressions found in any of the balloon definitions. Signed-off-by: Krisztian Litkey --- pkg/apis/config/v1alpha1/balloons-policy.go | 7 +++++++ .../v1alpha1/resmgr/policy/balloons/config.go | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/pkg/apis/config/v1alpha1/balloons-policy.go b/pkg/apis/config/v1alpha1/balloons-policy.go index 71fecd492..9622b6b43 100644 --- a/pkg/apis/config/v1alpha1/balloons-policy.go +++ b/pkg/apis/config/v1alpha1/balloons-policy.go @@ -35,3 +35,10 @@ func (c *BalloonsPolicy) PolicyConfig() interface{} { } return &c.Spec.Config } + +func (c *BalloonsPolicy) Validate() error { + if c == nil { + return nil + } + return c.Spec.Config.Validate() +} diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go index d3a8a717d..bf12ddd01 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go @@ -15,6 +15,7 @@ package balloons import ( + "errors" "strings" policy "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" @@ -223,3 +224,15 @@ func (p CPUPriority) Value() cpuallocator.CPUPriority { } return cpuallocator.PriorityNone } + +func (c *Config) Validate() error { + errs := []error{} + for _, blnDef := range c.BalloonDefs { + for _, expr := range blnDef.MatchExpressions { + if err := expr.Validate(); err != nil { + errs = append(errs, err) + } + } + } + return errors.Join(errs...) +} From 92a936a0d98e25615a2dc787f58d40b3463e383d Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Mon, 12 Feb 2024 23:23:10 +0200 Subject: [PATCH 3/4] docs: document matchExpressions for balloons. Signed-off-by: Krisztian Litkey --- docs/resource-policy/policy/balloons.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/resource-policy/policy/balloons.md b/docs/resource-policy/policy/balloons.md index 684bce91f..9bfcc350a 100644 --- a/docs/resource-policy/policy/balloons.md +++ b/docs/resource-policy/policy/balloons.md @@ -115,6 +115,15 @@ Balloons policy parameters: - `namespaces` is a list of namespaces (wildcards allowed) whose pods should be assigned to this balloon type, unless overridden by pod annotations. + - `matchExpressions` is a list of container match expressions. These + expressions are evaluated for all containers which have not been + assigned otherwise to other balloons. If an expression matches, + IOW it evaluates to true, the container gets assigned to this + balloon type. Container mach expressions have the same syntax and + semantics as the scope and match expressions in container affinity + annotations for the topology-aware policy. + See the [affinity documentation](./topology-aware.md#affinity-semantics) + for a detailed description of expressions. - `minBalloons` is the minimum number of balloons of this type that is always present, even if the balloons would not have any containers. The default is 0: if a balloon has no containers, it From 3b9fbf94cd2729fc02c63505ed58798596678908 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Tue, 13 Feb 2024 00:18:16 +0200 Subject: [PATCH 4/4] e2e: add balloons container match expression tests. Signed-off-by: Krisztian Litkey --- .../balloons/balloons-busybox.yaml.in | 3 +++ .../balloons/match-config.yaml | 27 +++++++++++++++++++ .../n4c16/test11-match-expression/code.var.sh | 24 +++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 test/e2e/policies.test-suite/balloons/match-config.yaml create mode 100644 test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh diff --git a/test/e2e/policies.test-suite/balloons/balloons-busybox.yaml.in b/test/e2e/policies.test-suite/balloons/balloons-busybox.yaml.in index 8bc86de3e..41413bf9c 100644 --- a/test/e2e/policies.test-suite/balloons/balloons-busybox.yaml.in +++ b/test/e2e/policies.test-suite/balloons/balloons-busybox.yaml.in @@ -8,6 +8,9 @@ metadata: "; fi) labels: app: ${NAME} + $(if [ -n "$POD_LABEL" ]; then echo " + $POD_LABEL + "; fi) spec: containers: $(for contnum in $(seq 1 ${CONTCOUNT}); do echo " diff --git a/test/e2e/policies.test-suite/balloons/match-config.yaml b/test/e2e/policies.test-suite/balloons/match-config.yaml new file mode 100644 index 000000000..29d68649e --- /dev/null +++ b/test/e2e/policies.test-suite/balloons/match-config.yaml @@ -0,0 +1,27 @@ +config: + reservedResources: + cpu: 750m + pinCPU: true + pinMemory: true + idleCPUClass: normal + allocatorTopologyBalancing: true + balloonTypes: + - name: special + matchExpressions: + - key: pod/labels/app.kubernetes.io/component + operator: Equals + values: [ "precious" ] + minCPUs: 2 + maxCPUs: 2 + - name: default + namespaces: + - default + minCPUs: 1 + maxCPUs: 1 + allocatorPriority: normal + reservedPoolNamespaces: + - kube-system + log: + debug: + - policy + - expression diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh new file mode 100644 index 000000000..70b065532 --- /dev/null +++ b/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh @@ -0,0 +1,24 @@ +# Test placing containers with and without annotations to correct balloons +# reserved and shared CPUs. + +helm-terminate +helm_config=${TEST_DIR}/../../match-config.yaml helm-launch balloons + +cleanup() { + vm-command "kubectl delete pods --all --now" + return 0 +} + +cleanup + +# pod0: run precious workload +POD_LABEL="app.kubernetes.io/component: precious" CONTCOUNT=1 create balloons-busybox +report allowed +verify 'len(cpus["pod0c0"]) == 2' + +# pod1: run ordinary workload +CONTCOUNT=1 create balloons-busybox +report allowed +verify 'len(cpus["pod1c0"]) == 1' + +cleanup