From 52ff0d42370052d3dc6913f83e97695f03558ec2 Mon Sep 17 00:00:00 2001 From: Github Date: Wed, 21 Jun 2023 11:00:42 +0000 Subject: [PATCH] Update documentation for devel Auto-generated from 19bbdd5 by 'update-gh-pages.sh' --- devel/.buildinfo | 2 +- devel/_sources/docs/index.rst.txt | 2 + devel/_sources/docs/introduction.md.txt | 6 + .../resource-policy/configuration.rst.txt | 7 + .../cpu-allocator.md.txt | 0 .../developers-guide/index.rst.txt | 1 + .../docs/resource-policy/index.rst.txt | 9 +- .../docs/resource-policy/installation.md.txt | 144 ++++++ .../policy/container-affinity.md.txt | 267 ----------- .../docs/resource-policy/policy/index.rst.txt | 12 +- .../policy/topology-aware.md.txt | 309 +++++++++++++ devel/_static/documentation_options.js | 2 +- devel/docs/contributing.html | 9 +- devel/docs/index.html | 15 +- devel/docs/introduction.html | 152 +++++++ devel/docs/resource-policy/configuration.html | 166 +++++++ .../developers-guide/architecture.html | 11 +- .../cpu-allocator.html | 29 +- .../developers-guide/e2e-test.html | 15 +- .../developers-guide/index.html | 12 +- .../developers-guide/testing.html | 11 +- .../developers-guide/unit-test.html | 11 +- devel/docs/resource-policy/index.html | 37 +- devel/docs/resource-policy/installation.html | 260 ++++++++++- devel/docs/resource-policy/introduction.html | 14 +- devel/docs/resource-policy/node-agent.html | 19 +- .../docs/resource-policy/policy/balloons.html | 29 +- .../policy/container-affinity.html | 430 ------------------ devel/docs/resource-policy/policy/index.html | 34 +- .../policy/topology-aware.html | 341 +++++++++++++- devel/docs/resource-policy/setup.html | 14 +- devel/genindex.html | 5 +- devel/objects.inv | Bin 1963 -> 2153 bytes devel/search.html | 5 +- devel/searchindex.js | 2 +- releases/.doctrees/environment.pickle | Bin 12679 -> 12679 bytes 36 files changed, 1518 insertions(+), 864 deletions(-) create mode 100644 devel/_sources/docs/introduction.md.txt create mode 100644 devel/_sources/docs/resource-policy/configuration.rst.txt rename devel/_sources/docs/resource-policy/{policy => developers-guide}/cpu-allocator.md.txt (100%) delete mode 100644 devel/_sources/docs/resource-policy/policy/container-affinity.md.txt create mode 100644 devel/docs/introduction.html create mode 100644 devel/docs/resource-policy/configuration.html rename devel/docs/resource-policy/{policy => developers-guide}/cpu-allocator.html (85%) delete mode 100644 devel/docs/resource-policy/policy/container-affinity.html diff --git a/devel/.buildinfo b/devel/.buildinfo index f18f38aee..0899cc82b 100644 --- a/devel/.buildinfo +++ b/devel/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 299a867199261978a71a6c7076261bc0 +config: 1dae0d9565992f1b681707cedf4e4d99 tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/devel/_sources/docs/index.rst.txt b/devel/_sources/docs/index.rst.txt index 4fcefd194..eb52560e6 100644 --- a/devel/_sources/docs/index.rst.txt +++ b/devel/_sources/docs/index.rst.txt @@ -7,6 +7,8 @@ Welcome to NRI Plugins documentation :maxdepth: 2 :caption: Contents: + introduction.md + resource-policy/index.rst contributing.md diff --git a/devel/_sources/docs/introduction.md.txt b/devel/_sources/docs/introduction.md.txt new file mode 100644 index 000000000..262835890 --- /dev/null +++ b/devel/_sources/docs/introduction.md.txt @@ -0,0 +1,6 @@ +# Introduction + +The NRI plugins is a collection of NRI (Node Resource Interface) based plugins +to manage various aspects of pod and container life cycle. +For example the [resource policy plugins](resource_policies) can be used to modify +the container resource allocation depending on available system resources. diff --git a/devel/_sources/docs/resource-policy/configuration.rst.txt b/devel/_sources/docs/resource-policy/configuration.rst.txt new file mode 100644 index 000000000..f5c5ee6a9 --- /dev/null +++ b/devel/_sources/docs/resource-policy/configuration.rst.txt @@ -0,0 +1,7 @@ +Configuration +############# +.. toctree:: + :maxdepth: 1 + + policy/index.rst + node-agent.md diff --git a/devel/_sources/docs/resource-policy/policy/cpu-allocator.md.txt b/devel/_sources/docs/resource-policy/developers-guide/cpu-allocator.md.txt similarity index 100% rename from devel/_sources/docs/resource-policy/policy/cpu-allocator.md.txt rename to devel/_sources/docs/resource-policy/developers-guide/cpu-allocator.md.txt diff --git a/devel/_sources/docs/resource-policy/developers-guide/index.rst.txt b/devel/_sources/docs/resource-policy/developers-guide/index.rst.txt index dc83815f6..05556c29a 100644 --- a/devel/_sources/docs/resource-policy/developers-guide/index.rst.txt +++ b/devel/_sources/docs/resource-policy/developers-guide/index.rst.txt @@ -5,3 +5,4 @@ Developer's Guide architecture.md testing.rst + cpu-allocator.md diff --git a/devel/_sources/docs/resource-policy/index.rst.txt b/devel/_sources/docs/resource-policy/index.rst.txt index 2fcb51f6c..0d08a7d1d 100644 --- a/devel/_sources/docs/resource-policy/index.rst.txt +++ b/devel/_sources/docs/resource-policy/index.rst.txt @@ -1,7 +1,5 @@ -.. NRI Resource Policy documentation master file - -Resource Policy Plugin -====================== +Resource Policy Plugins +======================= .. toctree:: :maxdepth: 2 @@ -10,7 +8,6 @@ Resource Policy Plugin introduction.md installation.md setup.md - policy/index.rst - node-agent.md + configuration.rst developers-guide/index.rst diff --git a/devel/_sources/docs/resource-policy/installation.md.txt b/devel/_sources/docs/resource-policy/installation.md.txt index 25267fe2b..fc0bf2d1a 100644 --- a/devel/_sources/docs/resource-policy/installation.md.txt +++ b/devel/_sources/docs/resource-policy/installation.md.txt @@ -1 +1,145 @@ # Installation + +This repository hosts a collection of plugins of various types, one of which is the resource +policy plugins. In this example, we will demonstrate the installation process for the topology-aware +plugin, which falls under the resource policy type. The installation methods outlined +here can be applied to any other plugin hosted in this repository, regardless of its type. + +Currently, there are two installation methods available. + +1. [Helm](#installing-the-helm-chart) +2. [Manual](#manual-installation) + +Regardless of the chosen installation method, the NRI plugin installation includes the +following components: DaemonSet, ConfigMap, CustomResourceDefinition, and RBAC-related objects. + +## Prerequisites + +- Container runtime: + - containerD: + - At least [containerd 1.7.0](https://github.com/containerd/containerd/releases/tag/v1.7.0) release version to use the NRI feature + - Enable NRI feature by following [these](TODO link) detailed instructions. + - CRI-O + - At least [v1.26.0](https://github.com/cri-o/cri-o/releases/tag/v1.26.0) release version to use the NRI feature + - Enable NRI feature by following [these](TODO link) detailed instructions. +- Kubernetes 1.24+ +- Helm 3.0.0+ + +## Installing the Helm Chart + +1. Clone the project to your local machine + ```sh + git clone https://github.com/containers/nri-plugins.git + ``` + +1. Navigate to the project directory + ```sh + cd nri-plugins + ``` + +1. Install the plugin using Helm. Replace release name with the desired name + for your Helm release. In this example, we named it as topology-aware. The +default values for topology-aware resource policy plugin are stored in +values.yaml file. If you wish to provide custom values to the Helm +chart, refer to the [table](#helm-parameters) below, which describes the +available parameters that can be modified before installation. + + ```sh + helm install topology-aware deployment/helm/resource-management-policies/topology-aware/ + ``` + +1. Verify the status of the daemonset to ensure that the plugin is running successfully + + ```bash + kubectl get daemonset -n kube-system nri-resource-policy + + NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE + nri-resource-policy 1 1 0 1 0 kubernetes.io/os=linux 4m33s + ``` + +That's it! You have now installed the topology-aware NRI resource policy plugin using Helm. + +## Uninstalling the Chart + +To uninstall plugin chart just deleting it with the release name is enough: + +```bash +helm delete topology-aware +``` + +Note: this removes DaemonSet, ConfigMap, CustomResourceDefinition, and RBAC-related objects associated with the chart. + +### Helm parameters + +The tables below present an overview of the parameters available for users to customize with their own values, along +with the default values, for the Topology-aware and Balloons plugins Helm charts. + +#### Topology-aware + +| Name | Default | Description | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------- | +| `namespace` | kube-system | cpecify the namespace to use for deploying DaemonSet, ConfigMap, CustomResourceDefinition, and RBAC-related objects | +| `image.name` | [ghcr.io/containers/nri-plugins/nri-resource-policy-topology-aware](ghcr.io/containers/nri-plugins/nri-resource-policy-topology-aware) | container image name | +| `image.tag` | unstable | container image tag | +| `image.pullPolicy` | Always | image pull policy | +| `quota.cpu` | 500m | cpu qouta for the Pod | +| `quota.memory` | 512Mi | memory qouta for the Pod | +| `metrics.port` | 8891 | metrics port on the container | +| `metrics.hostPort` | 8891 | metrics port to expose on the host | +| `config` | 
Active: balloons
ReservedResources:
  cpu: 750m
| plugin configuration data | + +#### Balloons + +| Name | Default | Description | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------- | +| `namespace` | kube-system | specify the namespace to use for deploying DaemonSet, ConfigMap, CustomResourceDefinition, and RBAC-related objects | +| `image.name` | [ghcr.io/containers/nri-plugins/nri-resource-policy-balloons](ghcr.io/containers/nri-plugins/nri-resource-policy-balloons) | container image name | +| `image.tag` | unstable | container image tag | +| `image.pullPolicy` | Always | image pull policy | +| `quota.cpu` | 500m | cpu qouta for the Pod | +| `quota.memory` | 512Mi | Memory qouta for the Pod | +| `metrics.port` | 8891 | metrics port on the container | +| `metrics.hostPort` | 8891 | metrics port to expose on the host | +| `config` | 
Active: topology-aware
ReservedResources:
  cpu: 750m
| plugin configuration data | + +## Manual installation + +For the manual installation we will be using templating tool to generate Kubernetes YAML manifests. +1. Clone the project to your local machine + ```sh + git clone https://github.com/containers/nri-plugins.git + ``` + +1. Navigate to the project directory + ```sh + cd nri-plugins + ``` + +1. If there are any specific configuration values you need to modify, navigate to the plugins [directory](https://github.com/containers/nri-plugins/tree/main/deployment/overlays) containing the Kustomization file and update the desired configuration values according to your environment in the Kustomization file. + +1. Use kustomize to generate the Kubernetes manifests for the desired plugin and apply the generated manifests to your Kubernetes cluster using kubectl. + + ```sh + kustomize build deployment/overlays/topology-aware/ | kubectl apply -f - + ``` + +1. Verify the status of the DaemonSet to ensure that the plugin is running successfully + + ```bash + kubectl get daemonset -n kube-system nri-resource-policy + + NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE + nri-resource-policy 1 1 0 1 0 kubernetes.io/os=linux 4m33s + ``` + +That's it! You have now installed the topology-aware NRI resource policy plugin using kutomize. + +## Manual uninstallation + +To uninstall plugin manifests you can run the following command: + +```sh +kustomize build deployment/overlays/topology-aware/ | kubectl delete -f - +``` + +Note: this removes DaemonSet, ConfigMap, CustomResourceDefinition, and RBAC-related objects associated with the chart. diff --git a/devel/_sources/docs/resource-policy/policy/container-affinity.md.txt b/devel/_sources/docs/resource-policy/policy/container-affinity.md.txt deleted file mode 100644 index 200ea26a9..000000000 --- a/devel/_sources/docs/resource-policy/policy/container-affinity.md.txt +++ /dev/null @@ -1,267 +0,0 @@ -# Container Affinity and Anti-Affinity - -## Introduction - -The topology-aware resource policy allow the user to give hints about how -particular containers should be *co-located* within a node. In particular these -hints express whether containers should be located *'close'* to each other or -*'far away'* from each other, in a hardware topology sense. - -Since these hints are interpreted always by a particular *policy implementation*, -the exact definitions of 'close' and 'far' are also somewhat *policy-specific*. -However as a general rule of thumb containers running - - - on CPUs within the *same NUMA nodes* are considered *'close'* to each other, - - on CPUs within *different NUMA nodes* in the *same socket* are *'farther'*, and - - on CPUs within *different sockets* are *'far'* from each other - -These hints are expressed by `container affinity annotations` on the Pod. -There are two types of affinities: - - - `affinity` (or `positive affinty`): cause affected containers to *pull* each other closer - - `anti-affinity` (or `negative affinity`): cause affected containers to *push* each other further away - -Policies try to place a container - - close to those the container has affinity towards - - far from those the container has anti-affinity towards. - -## Affinity Annotation Syntax - -*Affinities* are defined as the `resource-policy.nri.io/affinity` annotation. -*Anti-affinities* are defined as the `resource-policy.nri.io/anti-affinity` -annotation. They are specified in the `metadata` section of the `Pod YAML`, under -`annotations` as a dictionary, with each dictionary key being the name of the -*container* within the Pod to which the annotation belongs to. - -```yaml -metadata: - anotations: - resource-policy.nri.io/affinity: | - container1: - - scope: - key: key-ref - operator: op - values: - - value1 - ... - - valueN - match: - key: key-ref - operator: op - values: - - value1 - ... - - valueN - weight: w -``` - -An anti-affinity is defined similarly but using `resource-policy.nri.io/anti-affinity` -as the annotation key. - -```yaml -metadata: - anotations: - resource-policy.nri.io/anti-affinity: | - container1: - - scope: - key: key-ref - operator: op - values: - - value1 - ... - - valueN - match: - key: key-ref - operator: op - values: - - value1 - ... - - valueN - weight: w -``` - -## Affinity Semantics - -An affinity consists of three parts: - - - `scope expression`: defines which containers this affinity is evaluated against - - `match expression`: defines for which containers (within the scope) the affinity applies to - - `weight`: defines how *strong* a pull or a push the affinity causes - -*Affinities* are also sometimes referred to as *positive affinities* while -*anti-affinities* are referred to as *negative affinities*. The reason for this is -that the only difference between these are that affinities have a *positive weight* -while anti-affinities have a *negative weight*. - -The *scope* of an affinity defines the *bounding set of containers* the affinity can -apply to. The affinity *expression* is evaluated against the containers *in scope* and -it *selects the containers* the affinity really has an effect on. The *weight* specifies -whether the effect is a *pull* or a *push*. *Positive* weights cause a *pull* while -*negative* weights cause a *push*. Additionally, the *weight* specifies *how strong* the -push or the pull is. This is useful in situations where the policy needs to make some -compromises because an optimal placement is not possible. The weight then also acts as -a way to specify preferences of priorities between the various compromises: the heavier -the weight the stronger the pull or push and the larger the propbability that it will be -honored, if this is possible at all. - -The scope can be omitted from an affinity in which case it implies *Pod scope*, in other -words the scope of all containers that belong to the same Pod as the container for which -which the affinity is defined. - -The weight can also be omitted in which case it defaults to -1 for anti-affinities -and +1 for affinities. Weights are currently limited to the range [-1000,1000]. - -Both the affinity scope and the expression select containers, therefore they are identical. -Both of them are *expressions*. An expression consists of three parts: - - - key: specifies what *metadata* to pick from a container for evaluation - - operation (op): specifies what *logical operation* the expression evaluates - - values: a set of *strings* to evaluate the the value of the key against - -The supported keys are: - - - for pods: - - `name` - - `namespace` - - `qosclass` - - `labels/` - - `id` - - `uid` - - for containers: - - `pod/` - - `name` - - `namespace` - - `qosclass` - - `labels/` - - `tags/` - - `id` - -Essentially an expression defines a logical operation of the form (key op values). -Evaluating this logical expression will take the value of the key in which -either evaluates to true or false. -a boolean true/false result. Currently the following operations are supported: - - - `Equals`: equality, true if the *value of key* equals the single item in *values* - - `NotEqual`: inequality, true if the *value of key* is not equal to the single item in *values* - - `In`: membership, true if *value of key* equals to any among *values* - - `NotIn`: negated membership, true if the *value of key* is not equal to any among *values* - - `Exists`: true if the given *key* exists with any value - - `NotExists`: true if the given *key* does not exist - - `AlwaysTrue`: always evaluates to true, can be used to denote node-global scope (all containers) - - `Matches`: true if the *value of key* matches the globbing pattern in values - - `MatchesNot`: true if the *value of key* does not match the globbing pattern in values - - `MatchesAny`: true if the *value of key* matches any of the globbing patterns in values - - `MatchesNone`: true if the *value of key* does not match any of the globbing patterns in values - -The effective affinity between containers C_1 and C_2, A(C_1, C_2) is the sum of the -weights of all pairwise in-scope matching affinities W(C_1, C_2). To put it another way, -evaluating an affinity for a container C_1 is done by first using the scope (expression) -to determine which containers are in the scope of the affinity. Then, for each in-scope -container C_2 for which the match expression evaluates to true, taking the weight of the -affinity and adding it to the effective affinity A(C_1, C_2). - -Note that currently (for the topology-aware policy) this evaluation is asymmetric: -A(C_1, C_2) and A(C_2, C_1) can and will be different unless the affinity annotations are -crafted to prevent this (by making them fully symmetric). Moreover, A(C_1, C_2) is calculated -and taken into consideration during resource allocation for C_1, while A(C_2, C_1) -is calculated and taken into account during resource allocation for C_2. This might be -changed in a future version. - - -Currently affinity expressions lack support for boolean operators (and, or, not). -Sometimes this limitation can be overcome by using joint keys, especially with -matching operators. The joint key syntax allows joining the value of several keys -with a separator into a single value. A joint key can be specified in a simple or -full format: - - - simple: ``, this is equivalent to `:::` - - full: `` - -A joint key evaluates to the values of all the ``-separated subkeys joined by ``. -A non-existent subkey evaluates to the empty string. For instance the joint key - - `:pod/qosclass:pod/name:name` - -evaluates to - - `::` - -For existence operators, a joint key is considered to exist if any of its subkeys exists. - - -## Examples - -Put the container `peter` close to the container `sheep` but far away from the -container `wolf`. - -```yaml -metadata: - annotations: - resource-policy.nri.io/affinity: | - peter: - - match: - key: name - operator: Equals - values: - - sheep - weight: 5 - resource-policy.nri.io/anti-affinity: | - peter: - - match: - key: name - operator: Equals - values: - - wolf - weight: 5 -``` - -## Shorthand Notation - -There is an alternative shorthand syntax for what is considered to be the most common -case: defining affinities between containers within the same pod. With this notation -one needs to give just the names of the containers, like in the example below. - -```yaml - annotations: - resource-policy.nri.io/affinity: | - container3: [ container1 ] - resource-policy.nri.io/anti-affinity: | - container3: [ container2 ] - container4: [ container2, container3 ] -``` - - -This shorthand notation defines: - - `container3` having - - affinity (weight 1) to `container1` - - `anti-affinity` (weight -1) to `container2` - - `container4` having - - `anti-affinity` (weight -1) to `container2`, and `container3` - -The equivalent annotation in full syntax would be - -```yaml -metadata: - annotations: - resource-policy.nri.io/affinity: |+ - container3: - - match: - key: labels/io.kubernetes.container.name - operator: In - values: - - container1 - resource-policy.nri.io/anti-affinity: |+ - container3: - - match: - key: labels/io.kubernetes.container.name - operator: In - values: - - container2 - container4: - - match: - key: labels/io.kubernetes.container.name - operator: In - values: - - container2 - - container3 -``` diff --git a/devel/_sources/docs/resource-policy/policy/index.rst.txt b/devel/_sources/docs/resource-policy/policy/index.rst.txt index 35bd500de..bc4416a24 100644 --- a/devel/_sources/docs/resource-policy/policy/index.rst.txt +++ b/devel/_sources/docs/resource-policy/policy/index.rst.txt @@ -1,10 +1,18 @@ +.. _resource_policies: + Policies ######## +Currently there are two resource policies: + +The Topology Aware resource policy provides a nearly zero configuration resource +policy that allocates resources evenly in order to avoid the "noisy neighbor" problem. + +The Balloons resource policy allows user to allocate workloads to resources in a more +user controlled way. + .. toctree:: :maxdepth: 1 topology-aware.md balloons.md - container-affinity.md - cpu-allocator.md diff --git a/devel/_sources/docs/resource-policy/policy/topology-aware.md.txt b/devel/_sources/docs/resource-policy/policy/topology-aware.md.txt index 9904e891d..63edba758 100644 --- a/devel/_sources/docs/resource-policy/policy/topology-aware.md.txt +++ b/devel/_sources/docs/resource-policy/policy/topology-aware.md.txt @@ -430,3 +430,312 @@ metadata: prefer-reserved-cpus.resource-policy.nri.io/pod: "true" prefer-reserved-cpus.resource-policy.nri.io/container.special: "false" ``` + +## Allowing or denying mount/device paths via annotations + +User is able mark certain pods and containers to have allowed or denied +paths for mounts or devices. What this means is that when the system +is generating topology hints, it will consult this allowed / denied path +list to determine what hints are created. The deny path list is checked +first and then the allowed path list. This means that you can deny all +the mount/device paths and then allow only the needed ones for example. +User can either set the path with "prefix" (this is the default) or with +"glob" type. The "prefix" type means that the prefix of the mount/device +path is checked for matches. The "glob" type means that user is able to +put wildcards to the matched paths. + +For example: + +```yaml +metadata: + annotations: + allow.topologyhints.resource-policy.nri.io/pod: |+ + type: prefix + paths: + - /foo/bar/whitelisted-path1 + - /xy-zy/another-whitelisted-path1 + allow.topologyhints.resource-policy.nri.io/container.pod0c0: |+ + type: glob + paths: + - /whitelisted-path*2 + - /xy-zy/another-whitelisted-path2 + deny.topologyhints.resource-policy.nri.io: |+ + type: prefix + paths: + - /foo/bar/blacklisted-path3 + deny.topologyhints.resource-policy.nri.io/pod: |+ + type: glob + paths: + - /blacklisted-path*4 + deny.topologyhints.resource-policy.nri.io/container.pod0c1: |+ + type: prefix + paths: + - /foo/bar/blacklisted-path5 + - /xy-zy/another-blacklisted-path5 +``` + +## Container Affinity and Anti-Affinity + +### Introduction + +The topology-aware resource policy allow the user to give hints about how +particular containers should be *co-located* within a node. In particular these +hints express whether containers should be located *'close'* to each other or +*'far away'* from each other, in a hardware topology sense. + +Since these hints are interpreted always by a particular *policy implementation*, +the exact definitions of 'close' and 'far' are also somewhat *policy-specific*. +However as a general rule of thumb containers running + + - on CPUs within the *same NUMA nodes* are considered *'close'* to each other, + - on CPUs within *different NUMA nodes* in the *same socket* are *'farther'*, and + - on CPUs within *different sockets* are *'far'* from each other + +These hints are expressed by `container affinity annotations` on the Pod. +There are two types of affinities: + + - `affinity` (or `positive affinty`): cause affected containers to *pull* each other closer + - `anti-affinity` (or `negative affinity`): cause affected containers to *push* each other further away + +Policies try to place a container + - close to those the container has affinity towards + - far from those the container has anti-affinity towards. + +### Affinity Annotation Syntax + +*Affinities* are defined as the `resource-policy.nri.io/affinity` annotation. +*Anti-affinities* are defined as the `resource-policy.nri.io/anti-affinity` +annotation. They are specified in the `metadata` section of the `Pod YAML`, under +`annotations` as a dictionary, with each dictionary key being the name of the +*container* within the Pod to which the annotation belongs to. + +```yaml +metadata: + anotations: + resource-policy.nri.io/affinity: | + container1: + - scope: + key: key-ref + operator: op + values: + - value1 + ... + - valueN + match: + key: key-ref + operator: op + values: + - value1 + ... + - valueN + weight: w +``` + +An anti-affinity is defined similarly but using `resource-policy.nri.io/anti-affinity` +as the annotation key. + +```yaml +metadata: + anotations: + resource-policy.nri.io/anti-affinity: | + container1: + - scope: + key: key-ref + operator: op + values: + - value1 + ... + - valueN + match: + key: key-ref + operator: op + values: + - value1 + ... + - valueN + weight: w +``` + +### Affinity Semantics + +An affinity consists of three parts: + + - `scope expression`: defines which containers this affinity is evaluated against + - `match expression`: defines for which containers (within the scope) the affinity applies to + - `weight`: defines how *strong* a pull or a push the affinity causes + +*Affinities* are also sometimes referred to as *positive affinities* while +*anti-affinities* are referred to as *negative affinities*. The reason for this is +that the only difference between these are that affinities have a *positive weight* +while anti-affinities have a *negative weight*. + +The *scope* of an affinity defines the *bounding set of containers* the affinity can +apply to. The affinity *expression* is evaluated against the containers *in scope* and +it *selects the containers* the affinity really has an effect on. The *weight* specifies +whether the effect is a *pull* or a *push*. *Positive* weights cause a *pull* while +*negative* weights cause a *push*. Additionally, the *weight* specifies *how strong* the +push or the pull is. This is useful in situations where the policy needs to make some +compromises because an optimal placement is not possible. The weight then also acts as +a way to specify preferences of priorities between the various compromises: the heavier +the weight the stronger the pull or push and the larger the propbability that it will be +honored, if this is possible at all. + +The scope can be omitted from an affinity in which case it implies *Pod scope*, in other +words the scope of all containers that belong to the same Pod as the container for which +which the affinity is defined. + +The weight can also be omitted in which case it defaults to -1 for anti-affinities +and +1 for affinities. Weights are currently limited to the range [-1000,1000]. + +Both the affinity scope and the expression select containers, therefore they are identical. +Both of them are *expressions*. An expression consists of three parts: + + - key: specifies what *metadata* to pick from a container for evaluation + - operation (op): specifies what *logical operation* the expression evaluates + - values: a set of *strings* to evaluate the the value of the key against + +The supported keys are: + + - for pods: + - `name` + - `namespace` + - `qosclass` + - `labels/` + - `id` + - `uid` + - for containers: + - `pod/` + - `name` + - `namespace` + - `qosclass` + - `labels/` + - `tags/` + - `id` + +Essentially an expression defines a logical operation of the form (key op values). +Evaluating this logical expression will take the value of the key in which +either evaluates to true or false. +a boolean true/false result. Currently the following operations are supported: + + - `Equals`: equality, true if the *value of key* equals the single item in *values* + - `NotEqual`: inequality, true if the *value of key* is not equal to the single item in *values* + - `In`: membership, true if *value of key* equals to any among *values* + - `NotIn`: negated membership, true if the *value of key* is not equal to any among *values* + - `Exists`: true if the given *key* exists with any value + - `NotExists`: true if the given *key* does not exist + - `AlwaysTrue`: always evaluates to true, can be used to denote node-global scope (all containers) + - `Matches`: true if the *value of key* matches the globbing pattern in values + - `MatchesNot`: true if the *value of key* does not match the globbing pattern in values + - `MatchesAny`: true if the *value of key* matches any of the globbing patterns in values + - `MatchesNone`: true if the *value of key* does not match any of the globbing patterns in values + +The effective affinity between containers C_1 and C_2, A(C_1, C_2) is the sum of the +weights of all pairwise in-scope matching affinities W(C_1, C_2). To put it another way, +evaluating an affinity for a container C_1 is done by first using the scope (expression) +to determine which containers are in the scope of the affinity. Then, for each in-scope +container C_2 for which the match expression evaluates to true, taking the weight of the +affinity and adding it to the effective affinity A(C_1, C_2). + +Note that currently (for the topology-aware policy) this evaluation is asymmetric: +A(C_1, C_2) and A(C_2, C_1) can and will be different unless the affinity annotations are +crafted to prevent this (by making them fully symmetric). Moreover, A(C_1, C_2) is calculated +and taken into consideration during resource allocation for C_1, while A(C_2, C_1) +is calculated and taken into account during resource allocation for C_2. This might be +changed in a future version. + + +Currently affinity expressions lack support for boolean operators (and, or, not). +Sometimes this limitation can be overcome by using joint keys, especially with +matching operators. The joint key syntax allows joining the value of several keys +with a separator into a single value. A joint key can be specified in a simple or +full format: + + - simple: ``, this is equivalent to `:::` + - full: `` + +A joint key evaluates to the values of all the ``-separated subkeys joined by ``. +A non-existent subkey evaluates to the empty string. For instance the joint key + + `:pod/qosclass:pod/name:name` + +evaluates to + + `::` + +For existence operators, a joint key is considered to exist if any of its subkeys exists. + + +### Examples + +Put the container `peter` close to the container `sheep` but far away from the +container `wolf`. + +```yaml + resource-policy.nri.io/affinity: | + peter: + - match: + key: name + operator: Equals + values: + - sheep + weight: 5 + resource-policy.nri.io/anti-affinity: | + peter: + - match: + key: name + operator: Equals + values: + - wolf + weight: 5 +``` + +### Shorthand Notation + +There is an alternative shorthand syntax for what is considered to be the most common +case: defining affinities between containers within the same pod. With this notation +one needs to give just the names of the containers, like in the example below. + +```yaml + annotations: + resource-policy.nri.io/affinity: | + container3: [ container1 ] + resource-policy.nri.io/anti-affinity: | + container3: [ container2 ] + container4: [ container2, container3 ] +``` + + +This shorthand notation defines: + - `container3` having + - affinity (weight 1) to `container1` + - `anti-affinity` (weight -1) to `container2` + - `container4` having + - `anti-affinity` (weight -1) to `container2`, and `container3` + +The equivalent annotation in full syntax would be + +```yaml +metadata: + annotations: + resource-policy.nri.io/affinity: |+ + container3: + - match: + key: labels/io.kubernetes.container.name + operator: In + values: + - container1 + resource-policy.nri.io/anti-affinity: |+ + container3: + - match: + key: labels/io.kubernetes.container.name + operator: In + values: + - container2 + container4: + - match: + key: labels/io.kubernetes.container.name + operator: In + values: + - container2 + - container3 +``` diff --git a/devel/_static/documentation_options.js b/devel/_static/documentation_options.js index b63b28435..935b043a1 100644 --- a/devel/_static/documentation_options.js +++ b/devel/_static/documentation_options.js @@ -1,6 +1,6 @@ var DOCUMENTATION_OPTIONS = { URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), - VERSION: '0.0.0-262-gd42d018', + VERSION: '0.0.0-276-g19bbdd5', LANGUAGE: 'en', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/devel/docs/contributing.html b/devel/docs/contributing.html index ea25ca07f..db3c3c73d 100644 --- a/devel/docs/contributing.html +++ b/devel/docs/contributing.html @@ -4,7 +4,7 @@ - Contributing — NRI Plugins 0.0.0-262-gd42d018 documentation + Contributing — NRI Plugins 0.0.0-276-g19bbdd5 documentation + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Introduction

+

The NRI plugins is a collection of NRI (Node Resource Interface) based plugins +to manage various aspects of pod and container life cycle. +For example the resource policy plugins can be used to modify +the container resource allocation depending on available system resources.

+
+ + +
+
+ +
+
+
+
+ + +
+ + GitHub Pages + devel + + +
+
+
Versions
+
+
+
+ all releases +
+
+
+
+ + + + + + + \ No newline at end of file diff --git a/devel/docs/resource-policy/configuration.html b/devel/docs/resource-policy/configuration.html new file mode 100644 index 000000000..3c8b2b72c --- /dev/null +++ b/devel/docs/resource-policy/configuration.html @@ -0,0 +1,166 @@ + + + + + + + Configuration — NRI Plugins 0.0.0-276-g19bbdd5 documentation + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Configuration

+
+ +
+
+ + +
+
+ +
+
+
+
+ + +
+ + GitHub Pages + devel + + +
+
+
Versions
+
+
+
+ all releases +
+
+
+
+ + + + + + + \ No newline at end of file diff --git a/devel/docs/resource-policy/developers-guide/architecture.html b/devel/docs/resource-policy/developers-guide/architecture.html index ba9cf16d1..f58e9be7d 100644 --- a/devel/docs/resource-policy/developers-guide/architecture.html +++ b/devel/docs/resource-policy/developers-guide/architecture.html @@ -4,7 +4,7 @@ - Architecture — NRI Plugins 0.0.0-262-gd42d018 documentation + Architecture — NRI Plugins 0.0.0-276-g19bbdd5 documentation - - - - - - - - - - - - - - - -
- - -
- -
-
-
- -
-
-
-
- -
-

Container Affinity and Anti-Affinity

-
-

Introduction

-

The topology-aware resource policy allow the user to give hints about how -particular containers should be co-located within a node. In particular these -hints express whether containers should be located ‘close’ to each other or -‘far away’ from each other, in a hardware topology sense.

-

Since these hints are interpreted always by a particular policy implementation, -the exact definitions of ‘close’ and ‘far’ are also somewhat policy-specific. -However as a general rule of thumb containers running

-
    -

  • on CPUs within the same NUMA nodes are considered ‘close’ to each other,

    • -

  • on CPUs within different NUMA nodes in the same socket are ‘farther’, and

    • -

  • on CPUs within different sockets are ‘far’ from each other

    • -
-

These hints are expressed by container affinity annotations on the Pod. -There are two types of affinities:

-
    -

  • affinity (or positive affinty): cause affected containers to pull each other closer

    • -

  • anti-affinity (or negative affinity): cause affected containers to push each other further away

    • -
-

Policies try to place a container

-
    -

  • close to those the container has affinity towards

    • -

  • far from those the container has anti-affinity towards.

    • -
-
-
-

Affinity Annotation Syntax

-

Affinities are defined as the resource-policy.nri.io/affinity annotation. -Anti-affinities are defined as the resource-policy.nri.io/anti-affinity -annotation. They are specified in the metadata section of the Pod YAML, under -annotations as a dictionary, with each dictionary key being the name of the -container within the Pod to which the annotation belongs to.

-
metadata:
-  anotations:
-    resource-policy.nri.io/affinity: |
-      container1:
-        - scope:
-            key: key-ref
-            operator: op
-            values:
-            - value1
-            ...
-            - valueN
-          match:
-            key: key-ref
-            operator: op
-            values:
-            - value1
-            ...
-            - valueN
-          weight: w
-
-
-

An anti-affinity is defined similarly but using resource-policy.nri.io/anti-affinity -as the annotation key.

-
metadata:
-  anotations:
-    resource-policy.nri.io/anti-affinity: |
-      container1:
-        - scope:
-            key: key-ref
-            operator: op
-            values:
-            - value1
-            ...
-            - valueN
-          match:
-            key: key-ref
-            operator: op
-            values:
-            - value1
-            ...
-            - valueN
-          weight: w
-
-
-
-
-

Affinity Semantics

-

An affinity consists of three parts:

-
    -

  • scope expression: defines which containers this affinity is evaluated against

    • -

  • match expression: defines for which containers (within the scope) the affinity applies to

    • -

  • weight: defines how strong a pull or a push the affinity causes

    • -
-

Affinities are also sometimes referred to as positive affinities while -anti-affinities are referred to as negative affinities. The reason for this is -that the only difference between these are that affinities have a positive weight -while anti-affinities have a negative weight.

-

The scope of an affinity defines the bounding set of containers the affinity can -apply to. The affinity expression is evaluated against the containers in scope and -it selects the containers the affinity really has an effect on. The weight specifies -whether the effect is a pull or a push. Positive weights cause a pull while -negative weights cause a push. Additionally, the weight specifies how strong the -push or the pull is. This is useful in situations where the policy needs to make some -compromises because an optimal placement is not possible. The weight then also acts as -a way to specify preferences of priorities between the various compromises: the heavier -the weight the stronger the pull or push and the larger the propbability that it will be -honored, if this is possible at all.

-

The scope can be omitted from an affinity in which case it implies Pod scope, in other -words the scope of all containers that belong to the same Pod as the container for which -which the affinity is defined.

-

The weight can also be omitted in which case it defaults to -1 for anti-affinities -and +1 for affinities. Weights are currently limited to the range [-1000,1000].

-

Both the affinity scope and the expression select containers, therefore they are identical. -Both of them are expressions. An expression consists of three parts:

-
    -

  • key: specifies what metadata to pick from a container for evaluation

    • -

  • operation (op): specifies what logical operation the expression evaluates

    • -

  • values: a set of strings to evaluate the the value of the key against

    • -
-

The supported keys are:

-
    -

  • for pods:

    -
      -

    • name

      • -

    • namespace

      • -

    • qosclass

      • -

    • labels/<label-key>

      • -

    • id

      • -

    • uid

      • -
    -
    • -

  • for containers:

    -
      -

    • pod/<pod-key>

      • -

    • name

      • -

    • namespace

      • -

    • qosclass

      • -

    • labels/<label-key>

      • -

    • tags/<tag-key>

      • -

    • id

      • -
    -
    • -
-

Essentially an expression defines a logical operation of the form (key op values). -Evaluating this logical expression will take the value of the key in which -either evaluates to true or false. -a boolean true/false result. Currently the following operations are supported:

-
    -

  • Equals: equality, true if the value of key equals the single item in values

    • -

  • NotEqual: inequality, true if the value of key is not equal to the single item in values

    • -

  • In: membership, true if value of key equals to any among values

    • -

  • NotIn: negated membership, true if the value of key is not equal to any among values

    • -

  • Exists: true if the given key exists with any value

    • -

  • NotExists: true if the given key does not exist

    • -

  • AlwaysTrue: always evaluates to true, can be used to denote node-global scope (all containers)

    • -

  • Matches: true if the value of key matches the globbing pattern in values

    • -

  • MatchesNot: true if the value of key does not match the globbing pattern in values

    • -

  • MatchesAny: true if the value of key matches any of the globbing patterns in values

    • -

  • MatchesNone: true if the value of key does not match any of the globbing patterns in values

    • -
-

The effective affinity between containers C_1 and C_2, A(C_1, C_2) is the sum of the -weights of all pairwise in-scope matching affinities W(C_1, C_2). To put it another way, -evaluating an affinity for a container C_1 is done by first using the scope (expression) -to determine which containers are in the scope of the affinity. Then, for each in-scope -container C_2 for which the match expression evaluates to true, taking the weight of the -affinity and adding it to the effective affinity A(C_1, C_2).

-

Note that currently (for the topology-aware policy) this evaluation is asymmetric: -A(C_1, C_2) and A(C_2, C_1) can and will be different unless the affinity annotations are -crafted to prevent this (by making them fully symmetric). Moreover, A(C_1, C_2) is calculated -and taken into consideration during resource allocation for C_1, while A(C_2, C_1) -is calculated and taken into account during resource allocation for C_2. This might be -changed in a future version.

-

Currently affinity expressions lack support for boolean operators (and, or, not). -Sometimes this limitation can be overcome by using joint keys, especially with -matching operators. The joint key syntax allows joining the value of several keys -with a separator into a single value. A joint key can be specified in a simple or -full format:

-
    -

  • simple: <colon-separated-subkeys>, this is equivalent to :::<colon-separated-subkeys>

    • -

  • full: <ksep><vsep><ksep-separated-keylist>

    • -
-

A joint key evaluates to the values of all the <ksep>-separated subkeys joined by <vsep>. -A non-existent subkey evaluates to the empty string. For instance the joint key

-

:pod/qosclass:pod/name:name

-

evaluates to

-

<qosclass>:<pod name>:<container name>

-

For existence operators, a joint key is considered to exist if any of its subkeys exists.

-
-
-

Examples

-

Put the container peter close to the container sheep but far away from the -container wolf.

-
metadata:
-  annotations:
-    resource-policy.nri.io/affinity: |
-      peter:
-      - match:
-          key: name
-          operator: Equals
-          values:
-          - sheep
-        weight: 5
-    resource-policy.nri.io/anti-affinity: |
-      peter:
-      - match:
-          key: name
-          operator: Equals
-          values:
-          - wolf
-        weight: 5
-
-
-
-
-

Shorthand Notation

-

There is an alternative shorthand syntax for what is considered to be the most common -case: defining affinities between containers within the same pod. With this notation -one needs to give just the names of the containers, like in the example below.

-
  annotations:
-    resource-policy.nri.io/affinity: |
-      container3: [ container1 ]
-    resource-policy.nri.io/anti-affinity: |
-      container3: [ container2 ]
-      container4: [ container2, container3 ]
-
-
-

This shorthand notation defines:

-
    -

  • container3 having

    -
      -

    • affinity (weight 1) to container1

      • -

    • anti-affinity (weight -1) to container2

      • -
    -
    • -

  • container4 having

    -
      -

    • anti-affinity (weight -1) to container2, and container3

      • -
    -
    • -
-

The equivalent annotation in full syntax would be

-
metadata:
-  annotations:
-    resource-policy.nri.io/affinity: |+
-      container3:
-      - match:
-          key: labels/io.kubernetes.container.name
-          operator: In
-          values:
-          - container1
-    resource-policy.nri.io/anti-affinity: |+
-      container3:
-      - match:
-          key: labels/io.kubernetes.container.name
-          operator: In
-          values:
-          - container2
-      container4:
-      - match:
-          key: labels/io.kubernetes.container.name
-          operator: In
-          values:
-          - container2
-          - container3
-
-
-
-
- - -
-
- -
-
-
-
- - -
- - GitHub Pages - devel - - -
-
-
Versions
-
-
-
- all releases -
-
-
-
- - - - - - - \ No newline at end of file diff --git a/devel/docs/resource-policy/policy/index.html b/devel/docs/resource-policy/policy/index.html index 4c6a7601f..b690d5215 100644 --- a/devel/docs/resource-policy/policy/index.html +++ b/devel/docs/resource-policy/policy/index.html @@ -4,7 +4,7 @@ - Policies — NRI Plugins 0.0.0-262-gd42d018 documentation + Policies — NRI Plugins 0.0.0-276-g19bbdd5 documentation