Merge pull request #799 from SchSeba/rdma-subsytem-mode

Rdma subsytem mode

Makefile

@@ -226,7 +226,7 @@ test-e2e-k8s: export NAMESPACE=sriov-network-operator
 test-e2e-k8s: test-e2e
 
 test-bindata-scripts: fakechroot
-	fakechroot ./test/scripts/enable-kargs_test.sh
+	fakechroot ./test/scripts/kargs_test.sh
 
 test-%: generate manifests envtest
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir=/tmp -p path)" HOME="$(shell pwd)" go test ./$*/... -coverprofile cover-$*.out -coverpkg ./... -v

api/v1/sriovnetworknodestate_types.go

@@ -27,6 +27,7 @@ import (
 type SriovNetworkNodeStateSpec struct {
 	Interfaces Interfaces `json:"interfaces,omitempty"`
 	Bridges    Bridges    `json:"bridges,omitempty"`
+	System     System     `json:"system,omitempty"`
 }
 
 type Interfaces []Interface
@@ -114,10 +115,17 @@ type OVSUplinkConfigExt struct {
 	Interface OVSInterfaceConfig `json:"interface,omitempty"`
 }
 
+type System struct {
+	// +kubebuilder:validation:Enum=shared;exclusive
+	//RDMA subsystem. Allowed value "shared", "exclusive".
+	RdmaMode string `json:"rdmaMode,omitempty"`
+}
+
 // SriovNetworkNodeStateStatus defines the observed state of SriovNetworkNodeState
 type SriovNetworkNodeStateStatus struct {
 	Interfaces    InterfaceExts `json:"interfaces,omitempty"`
 	Bridges       Bridges       `json:"bridges,omitempty"`
+	System        System        `json:"system,omitempty"`
 	SyncStatus    string        `json:"syncStatus,omitempty"`
 	LastSyncError string        `json:"lastSyncError,omitempty"`
 }

api/v1/sriovnetworkpoolconfig_types.go

@@ -21,6 +21,10 @@ type SriovNetworkPoolConfigSpec struct {
 	// Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards,
 	// even if maxUnavailable is greater than one.
 	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
+
+	// +kubebuilder:validation:Enum=shared;exclusive
+	// RDMA subsystem. Allowed value "shared", "exclusive".
+	RdmaMode string `json:"rdmaMode,omitempty"`
 }
 
 type OvsHardwareOffloadConfig struct {

bindata/scripts/kargs.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+set -x
+
+command=$1
+shift
+declare -a kargs=( "$@" )
+ret=0
+args=$(chroot /host/ cat /proc/cmdline)
+
+if chroot /host/ test -f /run/ostree-booted ; then
+    for t in "${kargs[@]}";do
+        if [[ $command == "add" ]];then
+          if [[ $args != *${t}* ]];then
+              if chroot /host/ rpm-ostree kargs | grep -vq ${t}; then
+                  chroot /host/ rpm-ostree kargs --append ${t} > /dev/null 2>&1
+              fi
+              let ret++
+          fi
+        fi
+        if [[ $command == "remove" ]];then
+          if [[ $args == *${t}* ]];then
+                if chroot /host/ rpm-ostree kargs | grep -q ${t}; then
+                    chroot /host/ rpm-ostree kargs --delete ${t} > /dev/null 2>&1
+                fi
+                let ret++
+            fi
+        fi
+    done
+else
+    chroot /host/ which grubby > /dev/null 2>&1
+    # if grubby is not there, let's tell it
+    if [ $? -ne 0 ]; then
+        exit 127
+    fi
+    for t in "${kargs[@]}";do
+      if [[ $command == "add" ]];then
+        if [[ $args != *${t}* ]];then
+            if chroot /host/ grubby --info=DEFAULT | grep args | grep -vq ${t}; then
+                chroot /host/ grubby --update-kernel=DEFAULT --args=${t} > /dev/null 2>&1
+            fi
+            let ret++
+        fi
+      fi
+      if [[ $command == "remove" ]];then
+          if [[ $args == *${t}* ]];then
+            if chroot /host/ grubby --info=DEFAULT | grep args | grep -q ${t}; then
+                chroot /host/ grubby --update-kernel=DEFAULT --remove-args=${t} > /dev/null 2>&1
+            fi
+            let ret++
+          fi
+      fi
+    done
+fi
+
+echo $ret

config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml

@@ -174,6 +174,15 @@ spec:
                   - pciAddress
                   type: object
                 type: array
+              system:
+                properties:
+                  rdmaMode:
+                    description: RDMA subsystem. Allowed value "shared", "exclusive".
+                    enum:
+                    - shared
+                    - exclusive
+                    type: string
+                type: object
             type: object
           status:
             description: SriovNetworkNodeStateStatus defines the observed state of
@@ -335,6 +344,15 @@ spec:
                 type: string
               syncStatus:
                 type: string
+              system:
+                properties:
+                  rdmaMode:
+                    description: RDMA subsystem. Allowed value "shared", "exclusive".
+                    enum:
+                    - shared
+                    - exclusive
+                    type: string
+                type: object
             type: object
         type: object
     served: true

config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml

@@ -111,6 +111,12 @@ spec:
                       Name is the name of MachineConfigPool to be enabled with OVS hardware offload
                     type: string
                 type: object
+              rdmaMode:
+                description: RDMA subsystem. Allowed value "shared", "exclusive".
+                enum:
+                - shared
+                - exclusive
+                type: string
             type: object
           status:
             description: SriovNetworkPoolConfigStatus defines the observed state of

controllers/drain_controller.go

@@ -24,11 +24,8 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -48,13 +45,6 @@ import (
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
 )
 
-var (
-	oneNode     = intstr.FromInt32(1)
-	defaultNpcl = &sriovnetworkv1.SriovNetworkPoolConfig{Spec: sriovnetworkv1.SriovNetworkPoolConfigSpec{
-		MaxUnavailable: &oneNode,
-		NodeSelector:   &metav1.LabelSelector{}}}
-)
-
 type DrainReconcile struct {
 	client.Client
 	Scheme   *runtime.Scheme
@@ -346,94 +336,7 @@ func (dr *DrainReconcile) tryDrainNode(ctx context.Context, node *corev1.Node) (
 }
 
 func (dr *DrainReconcile) findNodePoolConfig(ctx context.Context, node *corev1.Node) (*sriovnetworkv1.SriovNetworkPoolConfig, []corev1.Node, error) {
-	logger := log.FromContext(ctx)
-	logger.Info("findNodePoolConfig():")
-	// get all the sriov network pool configs
-	npcl := &sriovnetworkv1.SriovNetworkPoolConfigList{}
-	err := dr.List(ctx, npcl)
-	if err != nil {
-		logger.Error(err, "failed to list sriovNetworkPoolConfig")
-		return nil, nil, err
-	}
-
-	selectedNpcl := []*sriovnetworkv1.SriovNetworkPoolConfig{}
-	nodesInPools := map[string]interface{}{}
-
-	for _, npc := range npcl.Items {
-		// we skip hw offload objects
-		if npc.Spec.OvsHardwareOffloadConfig.Name != "" {
-			continue
-		}
-
-		if npc.Spec.NodeSelector == nil {
-			npc.Spec.NodeSelector = &metav1.LabelSelector{}
-		}
-
-		selector, err := metav1.LabelSelectorAsSelector(npc.Spec.NodeSelector)
-		if err != nil {
-			logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", npc.Spec.NodeSelector)
-			return nil, nil, err
-		}
-
-		if selector.Matches(labels.Set(node.Labels)) {
-			selectedNpcl = append(selectedNpcl, npc.DeepCopy())
-		}
-
-		nodeList := &corev1.NodeList{}
-		err = dr.List(ctx, nodeList, &client.ListOptions{LabelSelector: selector})
-		if err != nil {
-			logger.Error(err, "failed to list all the nodes matching the pool with label selector from nodeSelector",
-				"machineConfigPoolName", npc,
-				"nodeSelector", npc.Spec.NodeSelector)
-			return nil, nil, err
-		}
-
-		for _, nodeName := range nodeList.Items {
-			nodesInPools[nodeName.Name] = nil
-		}
-	}
-
-	if len(selectedNpcl) > 1 {
-		// don't allow the node to be part of multiple pools
-		err = fmt.Errorf("node is part of more then one pool")
-		logger.Error(err, "multiple pools founded for a specific node", "numberOfPools", len(selectedNpcl), "pools", selectedNpcl)
-		return nil, nil, err
-	} else if len(selectedNpcl) == 1 {
-		// found one pool for our node
-		logger.V(2).Info("found sriovNetworkPool", "pool", *selectedNpcl[0])
-		selector, err := metav1.LabelSelectorAsSelector(selectedNpcl[0].Spec.NodeSelector)
-		if err != nil {
-			logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", selectedNpcl[0].Spec.NodeSelector)
-			return nil, nil, err
-		}
-
-		// list all the nodes that are also part of this pool and return them
-		nodeList := &corev1.NodeList{}
-		err = dr.List(ctx, nodeList, &client.ListOptions{LabelSelector: selector})
-		if err != nil {
-			logger.Error(err, "failed to list nodes using with label selector", "labelSelector", selector)
-			return nil, nil, err
-		}
-
-		return selectedNpcl[0], nodeList.Items, nil
-	} else {
-		// in this case we get all the nodes and remove the ones that already part of any pool
-		logger.V(1).Info("node doesn't belong to any pool, using default drain configuration with MaxUnavailable of one", "pool", *defaultNpcl)
-		nodeList := &corev1.NodeList{}
-		err = dr.List(ctx, nodeList)
-		if err != nil {
-			logger.Error(err, "failed to list all the nodes")
-			return nil, nil, err
-		}
-
-		defaultNodeLists := []corev1.Node{}
-		for _, nodeObj := range nodeList.Items {
-			if _, exist := nodesInPools[nodeObj.Name]; !exist {
-				defaultNodeLists = append(defaultNodeLists, nodeObj)
-			}
-		}
-		return defaultNpcl, defaultNodeLists, nil
-	}
+	return findNodePoolConfig(ctx, node, dr.Client)
 }
 
 // SetupWithManager sets up the controller with the Manager.