Deploy configuration daemon DS during config template reconcile

Signed-off-by: Alexander Maslennikov <[email protected]>

config/rbac/role.yaml

@@ -10,6 +10,8 @@ rules:
   - configmaps
   verbs:
   - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:

go.mod

@@ -7,10 +7,12 @@ toolchain go1.22.4
 require (
 	github.com/Mellanox/maintenance-operator/api v0.0.0-20240916123230-810ab7bb25f4
 	github.com/Mellanox/rdmamap v1.1.0
+	github.com/go-task/slim-sprig/v3 v3.0.0
 	github.com/jaypipes/ghw v0.12.0
 	github.com/jaypipes/pcidb v1.0.1
 	github.com/onsi/ginkgo/v2 v2.20.0
 	github.com/onsi/gomega v1.34.1
+	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.9.0
 	github.com/vishvananda/netlink v1.3.0
 	go.uber.org/zap v1.26.0
@@ -38,7 +40,6 @@ require (
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
-	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
@@ -56,7 +57,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.20.2 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect

internal/controller/nicconfigurationtemplate_controller.go

@@ -19,6 +19,7 @@ package controller
 import (
 	"context"
 	"fmt"
+	"os"
 	"reflect"
 	"slices"
 	"strings"
@@ -36,6 +37,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	v1alpha1 "github.com/Mellanox/nic-configuration-operator/api/v1alpha1"
+	"github.com/Mellanox/nic-configuration-operator/pkg/syncdaemon"
 )
 
 const nicConfigurationTemplateSyncEventName = "nic-configuration-template-sync-event"
@@ -54,6 +56,7 @@ type NicConfigurationTemplateReconciler struct {
 //+kubebuilder:rbac:groups=configuration.net.nvidia.com,resources=nicdevices,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=configuration.net.nvidia.com,resources=nicdevices/finalizers,verbs=update
 //+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;update;patch
+//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch
 //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get
 //+kubebuilder:rbac:groups="",resources=pods,verbs=list
 //+kubebuilder:rbac:groups="",resources=pods/eviction,verbs=create;delete;get;list;patch;update;watch
@@ -172,6 +175,12 @@ func (r *NicConfigurationTemplateReconciler) Reconcile(ctx context.Context, req
 		}
 	}
 
+	err = syncdaemon.SyncConfigDaemonObjs(ctx, r.Client, r.Scheme, os.Getenv("NAMESPACE"))
+	if err != nil {
+		log.Log.Error(err, "failed to sync ds")
+		return ctrl.Result{}, err
+	}
+
 	return ctrl.Result{}, nil
 }
 

pkg/consts/consts.go

@@ -82,4 +82,8 @@ const (
 	Mlx5ModuleVersionPath         = "/sys/bus/pci/drivers/mlx5_core/module/version"
 
 	FwConfigNotAppliedAfterRebootErrorMsg = "firmware configuration failed to apply after reboot"
+
+	ConfigDaemonManifestsPath = "./bindata/manifests/daemon"
+
+	OperatorConfigMapName = "nic-configuration-operator-config"
 )

pkg/render/render.go

@@ -0,0 +1,134 @@
+/*
+2024 NVIDIA CORPORATION & AFFILIATES
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package render
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	sprig "github.com/go-task/slim-sprig/v3"
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/util/yaml"
+)
+
+type RenderData struct {
+	Funcs template.FuncMap
+	Data  map[string]interface{}
+}
+
+func MakeRenderData() RenderData {
+	return RenderData{
+		Funcs: template.FuncMap{},
+		Data:  map[string]interface{}{},
+	}
+}
+
+// RenderDir will render all manifests in a directory, descending in to subdirectories
+// It will perform template substitutions based on the data supplied by the RenderData
+func RenderDir(manifestDir string, d *RenderData) ([]*unstructured.Unstructured, error) {
+	out := []*unstructured.Unstructured{}
+
+	if err := filepath.Walk(manifestDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		// Skip non-manifest files
+		if !(strings.HasSuffix(path, ".yml") || strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".json")) {
+			return nil
+		}
+
+		objs, err := RenderTemplate(path, d)
+		if err != nil {
+			return err
+		}
+		out = append(out, objs...)
+		return nil
+	}); err != nil {
+		return nil, errors.Wrap(err, "error rendering manifests")
+	}
+
+	return out, nil
+}
+
+// RenderTemplate reads, renders, and attempts to parse a yaml or
+// json file representing one or more k8s api objects
+func RenderTemplate(path string, d *RenderData) ([]*unstructured.Unstructured, error) {
+	rendered, err := renderTemplate(path, d)
+	if err != nil {
+		return nil, err
+	}
+
+	out := []*unstructured.Unstructured{}
+
+	// special case - if the entire file is whitespace, skip
+	if len(strings.TrimSpace(rendered.String())) == 0 {
+		return out, nil
+	}
+
+	decoder := yaml.NewYAMLOrJSONDecoder(rendered, 4096)
+	for {
+		u := unstructured.Unstructured{}
+		if err := decoder.Decode(&u); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, errors.Wrapf(err, "failed to unmarshal manifest %s", path)
+		}
+
+		if u.Object == nil {
+			continue
+		}
+
+		out = append(out, &u)
+	}
+
+	return out, nil
+}
+
+func renderTemplate(path string, d *RenderData) (*bytes.Buffer, error) {
+	tmpl := template.New(path).Option("missingkey=error")
+	if d.Funcs != nil {
+		tmpl.Funcs(d.Funcs)
+	}
+
+	// Add universal functions
+	tmpl.Funcs(sprig.TxtFuncMap())
+
+	source, err := os.ReadFile(path)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to read manifest %s", path)
+	}
+
+	if _, err := tmpl.Parse(string(source)); err != nil {
+		return nil, errors.Wrapf(err, "failed to parse manifest %s as template", path)
+	}
+
+	rendered := bytes.Buffer{}
+	if err := tmpl.Execute(&rendered, d.Data); err != nil {
+		return nil, errors.Wrapf(err, "failed to render manifest %s", path)
+	}
+
+	return &rendered, nil
+}

pkg/render/render_test.go

@@ -0,0 +1,132 @@
+/*
+2024 NVIDIA CORPORATION & AFFILIATES
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package render
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+)
+
+// TestRenderSimple tests rendering a single object with no templates
+func TestRenderSimple(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	d := MakeRenderData()
+
+	o1, err := RenderTemplate("testdata/manifests/simple.yaml", &d)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	g.Expect(o1).To(HaveLen(1))
+	expected := `
+{
+	"apiVersion": "v1",
+	"kind": "Pod",
+	"metadata": {
+		"name": "busybox1",
+		"namespace": "ns"
+	},
+	"spec": {
+		"containers": [
+			{
+  				"image": "busybox"
+			}
+		]
+	}
+}
+`
+	g.Expect(o1[0].MarshalJSON()).To(MatchJSON(expected))
+
+	// test that json parses the same
+	o2, err := RenderTemplate("testdata/manifests/simple.json", &d)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(o2).To(Equal(o1))
+}
+
+func TestRenderMultiple(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	p := "testdata/manifests/multiple.yaml"
+	d := MakeRenderData()
+
+	o, err := RenderTemplate(p, &d)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	g.Expect(o).To(HaveLen(3))
+
+	g.Expect(o[0].GetObjectKind().GroupVersionKind().String()).To(Equal("/v1, Kind=Pod"))
+	g.Expect(o[1].GetObjectKind().GroupVersionKind().String()).To(Equal("rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding"))
+	g.Expect(o[2].GetObjectKind().GroupVersionKind().String()).To(Equal("/v1, Kind=ConfigMap"))
+}
+
+func TestTemplate(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	p := "testdata/manifests/template.yaml"
+
+	// Test that missing variables are detected
+	d := MakeRenderData()
+	_, err := RenderTemplate(p, &d)
+	g.Expect(err).To(HaveOccurred())
+	g.Expect(err.Error()).To(HaveSuffix(`function "fname" not defined`))
+
+	// Set expected function (but not variable)
+	d.Funcs["fname"] = func(s string) string { return "test-" + s }
+	_, err = RenderTemplate(p, &d)
+	g.Expect(err).To(HaveOccurred())
+	g.Expect(err.Error()).To(HaveSuffix(`has no entry for key "Namespace"`))
+
+	// now we can render
+	d.Data["Namespace"] = "myns"
+	o, err := RenderTemplate(p, &d)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	g.Expect(o[0].GetName()).To(Equal("test-podname"))
+	g.Expect(o[0].GetNamespace()).To(Equal("myns"))
+}
+
+// TestTemplateWithEmptyObject tests the case where a file generates additional nil objects when rendered. An empty
+// object can also occur in the particular case shown in the testfile below when minus is missing at the end of the
+// first expression (i.e. {{- if .Enable }}).
+func TestTemplateWithEmptyObject(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	p := "testdata/manifests/template_with_empty_object.yaml"
+
+	d := MakeRenderData()
+	d.Data["Enable"] = true
+	o, err := RenderTemplate(p, &d)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	g.Expect(len(o)).To(Equal(2))
+	g.Expect(o[0].GetName()).To(Equal("pod1"))
+	g.Expect(o[0].GetNamespace()).To(Equal("namespace1"))
+	g.Expect(o[1].GetName()).To(Equal("pod2"))
+	g.Expect(o[1].GetNamespace()).To(Equal("namespace2"))
+}
+
+func TestRenderDir(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	d := MakeRenderData()
+	d.Funcs["fname"] = func(s string) string { return s }
+	d.Data["Namespace"] = "myns"
+	d.Data["Enable"] = true
+
+	o, err := RenderDir("testdata/manifests", &d)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(o).To(HaveLen(8))
+}

pkg/render/testdata/doc.txt

@@ -0,0 +1 @@
+Testing that documentation files are ignored.

pkg/render/testdata/manifests/multiple.yaml

@@ -0,0 +1,33 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: busybox1
+  namespace: ns
+spec:
+  containers:
+  - image: busybox
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: crb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io/v1
+  kind: ClusterRole
+  name: cr
+subjects:
+  - kind: ServiceAccount
+    name: sa
+    namespace: ns
+
+---
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cm
+  namespace: ns
+data:
+  key: val