Parallel NICs configuraion

api/v1/sriovoperatorconfig_types.go

@@ -43,6 +43,8 @@ type SriovOperatorConfigSpec struct {
 	// Default mode: daemon
 	// +kubebuilder:validation:Enum=daemon;systemd
 	ConfigurationMode ConfigurationModeType `json:"configurationMode,omitempty"`
+	// Flag to enable NICs configuration in a parallel on the same node
+	ParallelNicConfig bool `json:"parallelNicConfig,omitempty"`
 }
 
 // SriovOperatorConfigStatus defines the observed state of SriovOperatorConfig

cmd/sriov-network-config-daemon/service.go

@@ -113,9 +113,9 @@ func runServiceCmd(cmd *cobra.Command, args []string) error {
 			glog.Errorf("sriov-config-service: failed to discover sriov devices on the host: %v", err)
 			return fmt.Errorf("sriov-config-service: failed to discover sriov devices on the host:  %v", err)
 		}
-
+		// TODO(e0ne): read ParallelNicConfig from SriovOperatorConfig CR
 		// Create the generic plugin
-		configPlugin, err = generic.NewGenericPlugin(true)
+		configPlugin, err = generic.NewGenericPlugin(true, false)
 		if err != nil {
 			glog.Errorf("sriov-config-service: failed to create generic plugin %v", err)
 			return fmt.Errorf("sriov-config-service failed to create generic plugin %v", err)

cmd/sriov-network-config-daemon/start.go

@@ -52,9 +52,10 @@ var (
 	}
 
 	startOpts struct {
-		kubeconfig string
-		nodeName   string
-		systemd    bool
+		kubeconfig        string
+		nodeName          string
+		systemd           bool
+		parallelNicConfig bool
 	}
 )
 
@@ -63,6 +64,7 @@ func init() {
 	startCmd.PersistentFlags().StringVar(&startOpts.kubeconfig, "kubeconfig", "", "Kubeconfig file to access a remote cluster (testing only)")
 	startCmd.PersistentFlags().StringVar(&startOpts.nodeName, "node-name", "", "kubernetes node name daemon is managing")
 	startCmd.PersistentFlags().BoolVar(&startOpts.systemd, "use-systemd-service", false, "use config daemon in systemd mode")
+	startCmd.PersistentFlags().BoolVar(&startOpts.parallelNicConfig, "parallel-nic-config", false, "NICs configuration in a parallel on the same node")
 }
 
 func runStartCmd(cmd *cobra.Command, args []string) {
@@ -208,6 +210,7 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 		platformType,
 		startOpts.systemd,
 		devMode,
+		startOpts.parallelNicConfig,
 	).Run(stopCh, exitCh)
 	if err != nil {
 		glog.Errorf("failed to run daemon: %v", err)

config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml

@@ -72,6 +72,10 @@ spec:
                 maximum: 2
                 minimum: 0
                 type: integer
+              parallelNicConfig:
+                description: Flag to enable NICs configuration in a parallel on the
+                  same node
+                type: boolean
             type: object
           status:
             description: SriovOperatorConfigStatus defines the observed state of SriovOperatorConfig

deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml

@@ -72,6 +72,10 @@ spec:
                 maximum: 2
                 minimum: 0
                 type: integer
+              parallelNicConfig:
+                description: Flag to enable NICs configuration in a parallel on the
+                  same node
+                type: boolean
             type: object
           status:
             description: SriovOperatorConfigStatus defines the observed state of SriovOperatorConfig

pkg/daemon/daemon.go

@@ -69,6 +69,8 @@ type Daemon struct {
 
 	useSystemdService bool
 
+	parallelNicConfig bool
+
 	devMode bool
 
 	client snclientset.Interface
@@ -149,11 +151,13 @@ func New(
 	platformType utils.PlatformType,
 	useSystemdService bool,
 	devMode bool,
+	parallelNicConfig bool,
 ) *Daemon {
 	return &Daemon{
 		name:              nodeName,
 		platform:          platformType,
 		useSystemdService: useSystemdService,
+		parallelNicConfig: parallelNicConfig,
 		devMode:           devMode,
 		client:            client,
 		kubeClient:        kubeClient,
@@ -517,7 +521,7 @@ func (dn *Daemon) nodeStateSyncHandler() error {
 
 	// load plugins if it has not loaded
 	if len(dn.enabledPlugins) == 0 {
-		dn.enabledPlugins, err = enablePlugins(dn.platform, dn.useSystemdService, latestState)
+		dn.enabledPlugins, err = enablePlugins(dn.platform, dn.useSystemdService, dn.parallelNicConfig, latestState)
 		if err != nil {
 			glog.Errorf("nodeStateSyncHandler(): failed to enable vendor plugins error: %v", err)
 			return err

pkg/daemon/daemon_test.go

@@ -115,6 +115,7 @@ var _ = Describe("Config Daemon", func() {
 			utils.Baremetal,
 			false,
 			false,
+			false,
 		)
 
 		sut.enabledPlugins = map[string]plugin.VendorPlugin{generic.PluginName: &fake.FakePlugin{}}

pkg/daemon/plugin.go

@@ -28,7 +28,7 @@ var (
 	K8sPlugin         = k8splugin.NewK8sPlugin
 )
 
-func enablePlugins(platform utils.PlatformType, useSystemdService bool, ns *sriovnetworkv1.SriovNetworkNodeState) (map[string]plugin.VendorPlugin, error) {
+func enablePlugins(platform utils.PlatformType, useSystemdService, parallelNicConfig bool, ns *sriovnetworkv1.SriovNetworkNodeState) (map[string]plugin.VendorPlugin, error) {
 	glog.Infof("enableVendorPlugins(): enabling plugins")
 	enabledPlugins := map[string]plugin.VendorPlugin{}
 
@@ -54,7 +54,7 @@ func enablePlugins(platform utils.PlatformType, useSystemdService bool, ns *srio
 			}
 			enabledPlugins[k8sPlugin.Name()] = k8sPlugin
 		}
-		genericPlugin, err := GenericPlugin(false)
+		genericPlugin, err := GenericPlugin(false, parallelNicConfig)
 		if err != nil {
 			glog.Errorf("enableVendorPlugins(): failed to load the generic plugin error: %v", err)
 			return nil, err

pkg/plugins/generic/generic_plugin.go

@@ -47,19 +47,20 @@ type DriverState struct {
 type DriverStateMapType map[uint]*DriverState
 
 type GenericPlugin struct {
-	PluginName     string
-	SpecVersion    string
-	DesireState    *sriovnetworkv1.SriovNetworkNodeState
-	LastState      *sriovnetworkv1.SriovNetworkNodeState
-	DriverStateMap DriverStateMapType
-	RunningOnHost  bool
-	HostManager    host.HostManagerInterface
+	PluginName        string
+	SpecVersion       string
+	DesireState       *sriovnetworkv1.SriovNetworkNodeState
+	LastState         *sriovnetworkv1.SriovNetworkNodeState
+	DriverStateMap    DriverStateMapType
+	RunningOnHost     bool
+	ParallelNicConfig bool
+	HostManager       host.HostManagerInterface
 }
 
 const scriptsPath = "bindata/scripts/enable-kargs.sh"
 
 // Initialize our plugin and set up initial values
-func NewGenericPlugin(runningOnHost bool) (plugin.VendorPlugin, error) {
+func NewGenericPlugin(runningOnHost, parallelNicConfig bool) (plugin.VendorPlugin, error) {
 	driverStateMap := make(map[uint]*DriverState)
 	driverStateMap[Vfio] = &DriverState{
 		DriverName:     vfioPciDriver,
@@ -84,11 +85,12 @@ func NewGenericPlugin(runningOnHost bool) (plugin.VendorPlugin, error) {
 	}
 
 	return &GenericPlugin{
-		PluginName:     PluginName,
-		SpecVersion:    "1.0",
-		DriverStateMap: driverStateMap,
-		RunningOnHost:  runningOnHost,
-		HostManager:    host.NewHostManager(runningOnHost),
+		PluginName:        PluginName,
+		SpecVersion:       "1.0",
+		DriverStateMap:    driverStateMap,
+		RunningOnHost:     runningOnHost,
+		ParallelNicConfig: parallelNicConfig,
+		HostManager:       host.NewHostManager(runningOnHost),
 	}, nil
 }
 
@@ -166,7 +168,7 @@ func (p *GenericPlugin) Apply() error {
 		defer exit()
 	}
 
-	if err := utils.SyncNodeState(p.DesireState, pfsToSkip); err != nil {
+	if err := utils.SyncNodeState(p.DesireState, pfsToSkip, p.ParallelNicConfig); err != nil {
 		return err
 	}
 	p.LastState = &sriovnetworkv1.SriovNetworkNodeState{}

pkg/plugins/generic/generic_plugin_test.go

@@ -19,7 +19,7 @@ var _ = Describe("Generic plugin", func() {
 	var genericPlugin plugin.VendorPlugin
 	var err error
 	BeforeEach(func() {
-		genericPlugin, err = NewGenericPlugin(false)
+		genericPlugin, err = NewGenericPlugin(false, false)
 		Expect(err).ToNot(HaveOccurred())
 	})
 

pkg/utils/utils.go

@@ -12,6 +12,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
@@ -147,8 +148,71 @@ func DiscoverSriovDevices(withUnsupported bool) ([]sriovnetworkv1.InterfaceExt,
 }
 
 // SyncNodeState Attempt to update the node state to match the desired state
-func SyncNodeState(newState *sriovnetworkv1.SriovNetworkNodeState, pfsToConfig map[string]bool) error {
-	return ConfigSriovInterfaces(newState.Spec.Interfaces, newState.Status.Interfaces, pfsToConfig)
+func SyncNodeState(newState *sriovnetworkv1.SriovNetworkNodeState, pfsToConfig map[string]bool, parallelNicConfig bool) error {
+	if !parallelNicConfig {
+		return ConfigSriovInterfaces(newState.Spec.Interfaces, newState.Status.Interfaces, pfsToConfig)
+	}
+	return ConfigSriovInterfacesInParallel(newState.Spec.Interfaces, newState.Status.Interfaces, pfsToConfig)
+}
+
+func ConfigSriovInterfacesInParallel(interfaces []sriovnetworkv1.Interface, ifaceStatuses []sriovnetworkv1.InterfaceExt, pfsToConfig map[string]bool) error {
+	glog.V(2).Infof("ConfigSriovInterfacesInParallel(): start sriov configuration")
+	if IsKernelLockdownMode(true) && hasMellanoxInterfacesInSpec(ifaceStatuses, interfaces) {
+		glog.Warningf("cannot use mellanox devices when in kernel lockdown mode")
+		return fmt.Errorf("cannot use mellanox devices when in kernel lockdown mode")
+	}
+	// TODO(e0ne): store all errors in SriovNetworkNodeState
+	var result error
+	wg := sync.WaitGroup{}
+	for _, ifaceStatus := range ifaceStatuses {
+		configured := false
+		for _, iface := range interfaces {
+			if iface.PciAddress == ifaceStatus.PciAddress {
+				configured = true
+
+				if skip := pfsToConfig[iface.PciAddress]; skip {
+					break
+				}
+
+				if !NeedUpdate(&iface, &ifaceStatus) {
+					glog.V(2).Infof("syncNodeState(): no need update interface %s", iface.PciAddress)
+					break
+				}
+
+				wg.Add(1)
+				go func(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetworkv1.InterfaceExt) {
+					if err := configSriovDevice(iface, ifaceStatus); err != nil {
+						glog.Errorf("ConfigSriovInterfacesInParallel(): fail to configure sriov interface %s: %v. resetting interface.", iface.PciAddress, err)
+						result = err
+						if resetErr := resetSriovDevice(*ifaceStatus); resetErr != nil {
+							glog.Errorf("SyncNodeState(): fail to reset on error SR-IOV interface: %s", resetErr)
+							result = resetErr
+						}
+					}
+					wg.Done()
+				}(&iface, &ifaceStatus)
+
+				break
+			}
+		}
+		if !configured && ifaceStatus.NumVfs > 0 {
+			if skip := pfsToConfig[ifaceStatus.PciAddress]; skip {
+				continue
+			}
+
+			if err := resetSriovDevice(ifaceStatus); err != nil {
+				glog.V(2).Infof("ConfigSriovInterfacesInParallel(): reset failed %v", ifaceStatus.PciAddress)
+				result = err
+			}
+		}
+	}
+	wg.Wait()
+	if result != nil {
+		glog.Errorf("ConfigSriovInterfacesInParallel(): fail to configure sriov interface: %v", result)
+		return result
+	}
+	glog.V(2).Infof("ConfigSriovInterfacesInParallel(): sriov configuration finished")
+	return nil
 }
 
 func ConfigSriovInterfaces(interfaces []sriovnetworkv1.Interface, ifaceStatuses []sriovnetworkv1.InterfaceExt, pfsToConfig map[string]bool) error {
@@ -171,6 +235,7 @@ func ConfigSriovInterfaces(interfaces []sriovnetworkv1.Interface, ifaceStatuses
 					glog.V(2).Infof("syncNodeState(): no need update interface %s", iface.PciAddress)
 					break
 				}
+
 				if err = configSriovDevice(&iface, &ifaceStatus); err != nil {
 					glog.Errorf("SyncNodeState(): fail to configure sriov interface %s: %v. resetting interface.", iface.PciAddress, err)
 					if resetErr := resetSriovDevice(ifaceStatus); resetErr != nil {
@@ -415,6 +480,7 @@ func configSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetwor
 			return err
 		}
 	}
+	glog.V(2).Infof("configSriovDevice(): config interface %s completed", ifaceStatus.PciAddress)
 	return nil
 }
 
@@ -459,6 +525,7 @@ func setNetdevMTU(pciAddr string, mtu int) error {
 		glog.Warningf("setNetdevMTU(): fail to write mtu file after retrying: %v", err)
 		return err
 	}
+	glog.V(2).Infof("setNetdevMTU(): set MTU for device %s to %d completed", pciAddr, mtu)
 	return nil
 }