diff --git a/.gitignore b/.gitignore index 6696a5d59..540f9d33c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .cache/ .bash_history +/nvidia-dra-controller /nvidia-dra-plugin .idea [._]*.sw[a-p] diff --git a/cmd/nvidia-dra-controller/imex.go b/cmd/nvidia-dra-controller/imex.go new file mode 100644 index 000000000..d87ffff53 --- /dev/null +++ b/cmd/nvidia-dra-controller/imex.go @@ -0,0 +1,261 @@ +/* + * Copyright (c) 2024 NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "fmt" + "time" + + v1 "k8s.io/api/core/v1" + resourceapi "k8s.io/api/resource/v1alpha3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/dynamic-resource-allocation/resourceslice" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" +) + +const ( + DriverName = "gpu.nvidia.com" + ImexDomainLabel = "nvidia.com/gpu.imex-domain" + ImexChannelLimit = 128 +) + +type DriverResources resourceslice.DriverResources + +func StartIMEXManager(ctx context.Context, config *Config) error { + // Build a client set config + csconfig, err := config.flags.kubeClientConfig.NewClientSetConfig() + if err != nil { + return fmt.Errorf("error creating client set config: %w", err) + } + + // Create a new clientset + clientset, err := kubernetes.NewForConfig(csconfig) + if err != nil { + return fmt.Errorf("error creating dynamic client: %w", err) + } + + // Fetch the current Pod object + pod, err := clientset.CoreV1().Pods(config.flags.namespace).Get(ctx, config.flags.podName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("error fetching pod: %w", err) + } + + // Set the owner of the ResourceSlices we will create + owner := resourceslice.Owner{ + APIVersion: "v1", + Kind: "Pod", + Name: pod.Name, + UID: pod.UID, + } + + // Stream added/removed IMEX domains from nodes over time + klog.Info("Start streaming IMEX domains from nodes...") + addedDomainsCh, removedDomainsCh, err := streamImexDomains(ctx, clientset) + if err != nil { + return fmt.Errorf("error streaming IMEX domains: %w", err) + } + + // Add/Remove resource slices from IMEX domains as they come and go + klog.Info("Start publishing IMEX channels to ResourceSlices...") + err = manageResourceSlices(ctx, clientset, owner, addedDomainsCh, removedDomainsCh) + if err != nil { + return fmt.Errorf("error managing resource slices: %w", err) + } + + return nil +} + +// manageResourceSlices reacts to added and removed IMEX domains and triggers the creation / removal of resource slices accordingly. +func manageResourceSlices(ctx context.Context, clientset kubernetes.Interface, owner resourceslice.Owner, addedDomainsCh <-chan string, removedDomainsCh <-chan string) error { + driverResources := resourceslice.DriverResources{} + controller, err := resourceslice.StartController(ctx, clientset, DriverName, owner, &driverResources) + if err != nil { + return fmt.Errorf("error starting resource slice controller: %w", err) + } + + go func() { + for { + select { + case addedDomain := <-addedDomainsCh: + klog.Infof("Adding channels for new IMEX domain: %v", addedDomain) + newDriverResources := DriverResources(driverResources).DeepCopy() + newDriverResources.Pools[addedDomain] = generateImexChannelPool(addedDomain, ImexChannelLimit) + controller.Update(&newDriverResources) + driverResources = newDriverResources + case removedDomain := <-removedDomainsCh: + klog.Infof("Removing channels for removed IMEX domain: %v", removedDomain) + newDriverResources := DriverResources(driverResources).DeepCopy() + delete(newDriverResources.Pools, removedDomain) + controller.Update(&newDriverResources) + driverResources = newDriverResources + case <-ctx.Done(): + return + } + } + }() + + return nil +} + +// DeepCopy will perform a deep copy of the provided DriverResources. +func (d DriverResources) DeepCopy() resourceslice.DriverResources { + driverResources := resourceslice.DriverResources{ + Pools: make(map[string]resourceslice.Pool), + } + for p := range d.Pools { + driverResources.Pools[p] = generateImexChannelPool(p, ImexChannelLimit) + } + return driverResources +} + +// streamImexDomains returns two channels that streams imexDomans that are added and removed from nodes over time. +func streamImexDomains(ctx context.Context, clientset kubernetes.Interface) (<-chan string, <-chan string, error) { + // Create channels to stream IMEX domain ids that are added / removed + addedDomainCh := make(chan string) + removedDomainCh := make(chan string) + + // Use a map to track how many nodes are part of a given IMEX domain + nodesPerImexDomain := make(map[string]int) + + // Build a label selector to get all nodes with ImexDomainLabel set + requirement, err := labels.NewRequirement(ImexDomainLabel, selection.Exists, nil) + if err != nil { + return nil, nil, fmt.Errorf("error building label selector requirement: %w", err) + } + labelSelector := labels.NewSelector().Add(*requirement).String() + + // Create a shared informer factory for nodes + informerFactory := informers.NewSharedInformerFactoryWithOptions( + clientset, + time.Minute*10, // Resync period + informers.WithTweakListOptions(func(options *metav1.ListOptions) { + options.LabelSelector = labelSelector + }), + ) + nodeInformer := informerFactory.Core().V1().Nodes().Informer() + + // Set up event handlers for node events + _, err = nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + node := obj.(*v1.Node) // nolint:forcetypeassert + imexDomain := node.Labels[ImexDomainLabel] + if imexDomain != "" { + nodesPerImexDomain[imexDomain]++ + if nodesPerImexDomain[imexDomain] == 1 { + addedDomainCh <- imexDomain + } + } + }, + DeleteFunc: func(obj interface{}) { + node := obj.(*v1.Node) // nolint:forcetypeassert + imexDomain := node.Labels[ImexDomainLabel] + if imexDomain != "" { + nodesPerImexDomain[imexDomain]-- + if nodesPerImexDomain[imexDomain] == 0 { + removedDomainCh <- imexDomain + } + } + }, + UpdateFunc: func(oldObj, newObj interface{}) { + oldNode := oldObj.(*v1.Node) // nolint:forcetypeassert + newNode := newObj.(*v1.Node) // nolint:forcetypeassert + + oldImexDomain := oldNode.Labels[ImexDomainLabel] + newImexDomain := newNode.Labels[ImexDomainLabel] + + if oldImexDomain == newImexDomain { + return + } + if oldImexDomain != "" { + nodesPerImexDomain[oldImexDomain]-- + if nodesPerImexDomain[oldImexDomain] == 0 { + removedDomainCh <- oldImexDomain + } + } + if newImexDomain != "" { + nodesPerImexDomain[newImexDomain]++ + if nodesPerImexDomain[newImexDomain] == 1 { + addedDomainCh <- newImexDomain + } + } + }, + }) + if err != nil { + return nil, nil, fmt.Errorf("failed to create node informer: %w", err) + } + + // Start the informer and wait for it to sync + go informerFactory.Start(ctx.Done()) + + // Wait for the informer caches to sync + if !cache.WaitForCacheSync(ctx.Done(), nodeInformer.HasSynced) { + return nil, nil, fmt.Errorf("failed to sync informer caches") + } + + return addedDomainCh, removedDomainCh, nil +} + +// generateImexChannelPool generates the contents of a ResourceSlice pool for a given IMEX domain. +func generateImexChannelPool(imexDomain string, numChannels int) resourceslice.Pool { + // Generate dchannels from 0 to numChannels + var devices []resourceapi.Device + for i := 0; i < numChannels; i++ { + d := resourceapi.Device{ + Name: fmt.Sprintf("imex-channel-%d", i), + Basic: &resourceapi.BasicDevice{ + Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + "type": { + StringValue: ptr.To("imex-channel"), + }, + "channel": { + IntValue: ptr.To(int64(i)), + }, + }, + }, + } + devices = append(devices, d) + } + + // Put them in a pool named after the IMEX domain with the IMEX domain label as a node selector + pool := resourceslice.Pool{ + NodeSelector: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: ImexDomainLabel, + Operator: v1.NodeSelectorOpIn, + Values: []string{ + imexDomain, + }, + }, + }, + }, + }, + }, + Devices: devices, + } + + return pool +} diff --git a/cmd/nvidia-dra-controller/main.go b/cmd/nvidia-dra-controller/main.go new file mode 100644 index 000000000..e11b3dc71 --- /dev/null +++ b/cmd/nvidia-dra-controller/main.go @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2024 NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "net" + "net/http" + "net/http/pprof" + "os" + "path" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/urfave/cli/v2" + + "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/klog/v2" + + _ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration + _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration + _ "k8s.io/component-base/metrics/prometheus/workqueue" // register work queues in the default legacy registry + + "github.com/NVIDIA/k8s-dra-driver/internal/info" + "github.com/NVIDIA/k8s-dra-driver/pkg/flags" +) + +type Flags struct { + kubeClientConfig flags.KubeClientConfig + loggingConfig *flags.LoggingConfig + + podName string + namespace string + + httpEndpoint string + metricsPath string + profilePath string +} + +type Config struct { + flags *Flags + clientSets flags.ClientSets + mux *http.ServeMux +} + +func main() { + if err := newApp().Run(os.Args); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} + +func newApp() *cli.App { + flags := &Flags{ + loggingConfig: flags.NewLoggingConfig(), + } + cliFlags := []cli.Flag{ + &cli.StringFlag{ + Name: "pod-name", + Usage: "The name of the pod this controller is running in.", + Required: true, + Destination: &flags.podName, + EnvVars: []string{"POD_NAME"}, + }, + &cli.StringFlag{ + Name: "namespace", + Usage: "The namespace of the pod this controller is running in.", + Value: "default", + Destination: &flags.namespace, + EnvVars: []string{"NAMESPACE"}, + }, + &cli.StringFlag{ + Category: "HTTP server:", + Name: "http-endpoint", + Usage: "The TCP network `address` where the HTTP server for diagnostics, including pprof and metrics will listen (example: `:8080`). The default is the empty string, which means the server is disabled.", + Destination: &flags.httpEndpoint, + EnvVars: []string{"HTTP_ENDPOINT"}, + }, + &cli.StringFlag{ + Category: "HTTP server:", + Name: "metrics-path", + Usage: "The HTTP `path` where Prometheus metrics will be exposed, disabled if empty.", + Value: "/metrics", + Destination: &flags.metricsPath, + EnvVars: []string{"METRICS_PATH"}, + }, + &cli.StringFlag{ + Category: "HTTP server:", + Name: "pprof-path", + Usage: "The HTTP `path` where pprof profiling will be available, disabled if empty.", + Destination: &flags.profilePath, + EnvVars: []string{"PPROF_PATH"}, + }, + } + + cliFlags = append(cliFlags, flags.kubeClientConfig.Flags()...) + cliFlags = append(cliFlags, flags.loggingConfig.Flags()...) + + app := &cli.App{ + Name: "nvidia-dra-controller", + Usage: "nvidia-dra-controller implements a DRA driver controller for NVIDIA GPUs.", + ArgsUsage: " ", + HideHelpCommand: true, + Flags: cliFlags, + Before: func(c *cli.Context) error { + if c.Args().Len() > 0 { + return fmt.Errorf("arguments not supported: %v", c.Args().Slice()) + } + return flags.loggingConfig.Apply() + }, + Action: func(c *cli.Context) error { + ctx := c.Context + mux := http.NewServeMux() + + clientSets, err := flags.kubeClientConfig.NewClientSets() + if err != nil { + return fmt.Errorf("create client: %w", err) + } + + config := &Config{ + mux: mux, + flags: flags, + clientSets: clientSets, + } + + if flags.httpEndpoint != "" { + err = SetupHTTPEndpoint(config) + if err != nil { + return fmt.Errorf("create http endpoint: %w", err) + } + } + + err = StartIMEXManager(ctx, config) + if err != nil { + return fmt.Errorf("start IMEX manager: %w", err) + } + + <-ctx.Done() + + return nil + }, + Version: info.GetVersionString(), + } + + // We remove the -v alias for the version flag so as to not conflict with the -v flag used for klog. + f, ok := cli.VersionFlag.(*cli.BoolFlag) + if ok { + f.Aliases = nil + } + + return app +} + +func SetupHTTPEndpoint(config *Config) error { + if config.flags.metricsPath != "" { + // To collect metrics data from the metric handler itself, we + // let it register itself and then collect from that registry. + reg := prometheus.NewRegistry() + gatherers := prometheus.Gatherers{ + // Include Go runtime and process metrics: + // https://github.com/kubernetes/kubernetes/blob/9780d88cb6a4b5b067256ecb4abf56892093ee87/staging/src/k8s.io/component-base/metrics/legacyregistry/registry.go#L46-L49 + legacyregistry.DefaultGatherer, + } + gatherers = append(gatherers, reg) + + actualPath := path.Join("/", config.flags.metricsPath) + klog.InfoS("Starting metrics", "path", actualPath) + // This is similar to k8s.io/component-base/metrics HandlerWithReset + // except that we gather from multiple sources. + config.mux.Handle(actualPath, + promhttp.InstrumentMetricHandler( + reg, + promhttp.HandlerFor(gatherers, promhttp.HandlerOpts{}))) + } + + if config.flags.profilePath != "" { + actualPath := path.Join("/", config.flags.profilePath) + klog.InfoS("Starting profiling", "path", actualPath) + config.mux.HandleFunc(actualPath, pprof.Index) + config.mux.HandleFunc(path.Join(actualPath, "cmdline"), pprof.Cmdline) + config.mux.HandleFunc(path.Join(actualPath, "profile"), pprof.Profile) + config.mux.HandleFunc(path.Join(actualPath, "symbol"), pprof.Symbol) + config.mux.HandleFunc(path.Join(actualPath, "trace"), pprof.Trace) + } + + listener, err := net.Listen("tcp", config.flags.httpEndpoint) + if err != nil { + return fmt.Errorf("listen on HTTP endpoint: %w", err) + } + + go func() { + klog.InfoS("Starting HTTP server", "endpoint", config.flags.httpEndpoint) + err := http.Serve(listener, config.mux) + if err != nil { + klog.ErrorS(err, "HTTP server failed") + klog.FlushAndExit(klog.ExitFlushTimeout, 1) + } + }() + + return nil +} diff --git a/demo/clusters/kind/install-dra-driver.sh b/demo/clusters/kind/install-dra-driver.sh index 4533a7a7f..fc2aecc4b 100755 --- a/demo/clusters/kind/install-dra-driver.sh +++ b/demo/clusters/kind/install-dra-driver.sh @@ -29,8 +29,6 @@ helm upgrade -i --create-namespace --namespace nvidia-dra-driver nvidia ${PROJEC ${NVIDIA_DRIVER_ROOT:+--set nvidiaDriverRoot=${NVIDIA_DRIVER_ROOT}} \ --wait -kubectl apply -f "${CURRENT_DIR}/scripts/imex-resourceslice.yaml" - set +x printf '\033[0;32m' echo "Driver installation complete:" diff --git a/demo/clusters/kind/scripts/imex-resourceslice.yaml b/demo/clusters/kind/scripts/imex-resourceslice.yaml deleted file mode 100644 index cac7e2fd4..000000000 --- a/demo/clusters/kind/scripts/imex-resourceslice.yaml +++ /dev/null @@ -1,187 +0,0 @@ ---- -apiVersion: resource.k8s.io/v1alpha3 -kind: ResourceSlice -metadata: - name: imex-domain-0f884867-ba2f-4294-9155-b495ff367eea-1 -spec: - devices: - - name: imex-channel-0 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 0 - - name: imex-channel-1 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 1 - - name: imex-channel-2 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 2 - - name: imex-channel-3 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 3 - - name: imex-channel-4 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 4 - - name: imex-channel-5 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 5 - - name: imex-channel-6 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 6 - - name: imex-channel-7 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 7 - - name: imex-channel-8 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 8 - - name: imex-channel-9 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 9 - driver: gpu.nvidia.com - nodeSelector: - nodeSelectorTerms: - - matchExpressions: - - key: "nvidia.com/gpu.clusteruuid" - operator: In - values: - - "0f884867-ba2f-4294-9155-b495ff367eea" - - key: "nvidia.com/gpu.cliqueid" - operator: In - values: - - "1" - pool: - generation: 0 - name: imex-domain-0f884867-ba2f-4294-9155-b495ff367eea-1 - resourceSliceCount: 1 - ---- -apiVersion: resource.k8s.io/v1alpha3 -kind: ResourceSlice -metadata: - name: imex-domain-0f884867-ba2f-4294-9155-b495ff367eea-2 -spec: - devices: - - name: imex-channel-0 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 0 - - name: imex-channel-1 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 1 - - name: imex-channel-2 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 2 - - name: imex-channel-3 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 3 - - name: imex-channel-4 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 4 - - name: imex-channel-5 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 5 - - name: imex-channel-6 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 6 - - name: imex-channel-7 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 7 - - name: imex-channel-8 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 8 - - name: imex-channel-9 - basic: - attributes: - type: - string: "imex-channel" - channel: - int: 9 - driver: gpu.nvidia.com - nodeSelector: - nodeSelectorTerms: - - matchExpressions: - - key: "nvidia.com/gpu.clusteruuid" - operator: In - values: - - "0f884867-ba2f-4294-9155-b495ff367eea" - - key: "nvidia.com/gpu.cliqueid" - operator: In - values: - - "2" - pool: - generation: 0 - name: imex-domain-0f884867-ba2f-4294-9155-b495ff367eea-2 - resourceSliceCount: 1 diff --git a/deployments/container/Dockerfile.ubi8 b/deployments/container/Dockerfile.ubi8 index 3c0e83265..0f8f76980 100644 --- a/deployments/container/Dockerfile.ubi8 +++ b/deployments/container/Dockerfile.ubi8 @@ -67,6 +67,7 @@ LABEL org.opencontainers.image.description "NVIDIA GPU DRA driver for Kubernetes RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE +COPY --from=build /artifacts/nvidia-dra-controller /usr/bin/nvidia-dra-controller COPY --from=build /artifacts/nvidia-dra-plugin /usr/bin/nvidia-dra-plugin COPY --from=build /build/templates /templates diff --git a/deployments/container/Dockerfile.ubuntu b/deployments/container/Dockerfile.ubuntu index 7806c3b29..1d98a2a1f 100644 --- a/deployments/container/Dockerfile.ubuntu +++ b/deployments/container/Dockerfile.ubuntu @@ -67,6 +67,7 @@ LABEL org.opencontainers.image.description "NVIDIA GPU DRA driver for Kubernetes RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE +COPY --from=build /artifacts/nvidia-dra-controller /usr/bin/nvidia-dra-controller COPY --from=build /artifacts/nvidia-dra-plugin /usr/bin/nvidia-dra-plugin COPY --from=build /build/templates /templates diff --git a/deployments/helm/k8s-dra-driver/templates/controller.yaml b/deployments/helm/k8s-dra-driver/templates/controller.yaml new file mode 100644 index 000000000..b8f77a291 --- /dev/null +++ b/deployments/helm/k8s-dra-driver/templates/controller.yaml @@ -0,0 +1,62 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "k8s-dra-driver.fullname" . }}-controller + namespace: {{ include "k8s-dra-driver.namespace" . }} + labels: + {{- include "k8s-dra-driver.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "k8s-dra-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.controller.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "k8s-dra-driver.templateLabels" . | nindent 8 }} + spec: + {{- if .Values.controller.priorityClassName }} + priorityClassName: {{ .Values.controller.priorityClassName }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "k8s-dra-driver.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} + containers: + - name: controller + securityContext: + {{- toYaml .Values.controller.containers.controller.securityContext | nindent 10 }} + image: {{ include "k8s-dra-driver.fullimage" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["nvidia-dra-controller", "-v", "6"] + resources: + {{- toYaml .Values.controller.containers.controller.resources | nindent 10 }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.controller.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controller.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controller.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }}