From 4e0b66181d9870b8258d657c890d9ac1fc5c8180 Mon Sep 17 00:00:00 2001 From: Sebastian Sch Date: Tue, 20 Jun 2023 15:40:53 +0300 Subject: [PATCH] Create the new systemd cmd and implementation changes in the daemon pkg Signed-off-by: Sebastian Sch --- cmd/sriov-network-config-daemon/main.go | 15 ++ cmd/sriov-network-config-daemon/service.go | 182 +++++++++++++ cmd/sriov-network-config-daemon/start.go | 23 +- pkg/consts/constants.go | 36 +-- pkg/daemon/daemon.go | 189 +++++++++----- pkg/daemon/daemon_test.go | 4 +- pkg/daemon/plugin.go | 9 +- pkg/daemon/writer.go | 7 +- pkg/plugins/generic/generic_plugin.go | 23 +- pkg/plugins/generic/generic_plugin_test.go | 2 +- pkg/plugins/k8s/k8s_plugin.go | 120 +++++++-- pkg/plugins/virtual/virtual_plugin.go | 18 +- pkg/service/utils.go | 3 +- pkg/systemd/systemd.go | 288 +++++++++++++++++++++ 14 files changed, 797 insertions(+), 122 deletions(-) create mode 100644 cmd/sriov-network-config-daemon/service.go create mode 100644 pkg/systemd/systemd.go diff --git a/cmd/sriov-network-config-daemon/main.go b/cmd/sriov-network-config-daemon/main.go index 07719701b..3ceeed3ae 100644 --- a/cmd/sriov-network-config-daemon/main.go +++ b/cmd/sriov-network-config-daemon/main.go @@ -1,3 +1,18 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package main import ( diff --git a/cmd/sriov-network-config-daemon/service.go b/cmd/sriov-network-config-daemon/service.go new file mode 100644 index 000000000..b300254ac --- /dev/null +++ b/cmd/sriov-network-config-daemon/service.go @@ -0,0 +1,182 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "errors" + "flag" + "fmt" + "os" + + "github.com/golang/glog" + "github.com/spf13/cobra" + + sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" + plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/generic" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins/virtual" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/systemd" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/version" +) + +var ( + serviceCmd = &cobra.Command{ + Use: "service", + Short: "Starts SR-IOV service Config", + Long: "", + RunE: runServiceCmd, + } +) + +func init() { + rootCmd.AddCommand(serviceCmd) +} + +func runServiceCmd(cmd *cobra.Command, args []string) error { + flag.Set("logtostderr", "true") + flag.Parse() + + // To help debugging, immediately log version + glog.V(2).Infof("Version: %+v", version.Version) + + glog.V(0).Info("Starting sriov-config-service") + supportedNicIds, err := systemd.ReadSriovSupportedNics() + if err != nil { + glog.Errorf("failed to read list of supported nic ids") + sriovResult := &systemd.SriovResult{ + SyncStatus: "Failed", + LastSyncError: fmt.Sprintf("failed to read list of supported nic ids: %v", err), + } + err = systemd.WriteSriovResult(sriovResult) + if err != nil { + glog.Errorf("sriov-config-service failed to write sriov result file with content %v error: %v", *sriovResult, err) + return fmt.Errorf("sriov-config-service failed to write sriov result file with content %v error: %v", *sriovResult, err) + } + return fmt.Errorf("sriov-config-service failed to read list of supported nic ids: %v", err) + } + sriovv1.InitNicIDMapFromList(supportedNicIds) + + nodeStateSpec, err := systemd.ReadConfFile() + if err != nil { + if _, err := os.Stat(systemd.SriovSystemdConfigPath); !errors.Is(err, os.ErrNotExist) { + glog.Errorf("failed to read the sriov configuration file in path %s: %v", systemd.SriovSystemdConfigPath, err) + sriovResult := &systemd.SriovResult{ + SyncStatus: "Failed", + LastSyncError: fmt.Sprintf("failed to read the sriov configuration file in path %s: %v", systemd.SriovSystemdConfigPath, err), + } + err = systemd.WriteSriovResult(sriovResult) + if err != nil { + glog.Errorf("sriov-config-service failed to write sriov result file with content %v error: %v", *sriovResult, err) + return fmt.Errorf("sriov-config-service failed to write sriov result file with content %v error: %v", *sriovResult, err) + } + } + + nodeStateSpec = &systemd.SriovConfig{ + Spec: sriovv1.SriovNetworkNodeStateSpec{}, + UnsupportedNics: false, + PlatformType: utils.Baremetal, + } + } + + glog.V(2).Infof("sriov-config-service read config: %v", nodeStateSpec) + + // Load kernel modules + hostManager := host.NewHostManager(true) + _, err = hostManager.TryEnableRdma() + if err != nil { + glog.Warningf("failed to enable RDMA: %v", err) + } + hostManager.TryEnableTun() + hostManager.TryEnableVhostNet() + + var configPlugin plugin.VendorPlugin + var ifaceStatuses []sriovv1.InterfaceExt + if nodeStateSpec.PlatformType == utils.Baremetal { + // Bare metal support + ifaceStatuses, err = utils.DiscoverSriovDevices(nodeStateSpec.UnsupportedNics) + if err != nil { + glog.Errorf("sriov-config-service: failed to discover sriov devices on the host: %v", err) + return fmt.Errorf("sriov-config-service: failed to discover sriov devices on the host: %v", err) + } + + // Create the generic plugin + configPlugin, err = generic.NewGenericPlugin(true) + if err != nil { + glog.Errorf("sriov-config-service: failed to create generic plugin %v", err) + return fmt.Errorf("sriov-config-service failed to create generic plugin %v", err) + } + } else if nodeStateSpec.PlatformType == utils.VirtualOpenStack { + // Openstack support + metaData, networkData, err := utils.GetOpenstackData(false) + if err != nil { + glog.Errorf("sriov-config-service: failed to read OpenStack data: %v", err) + return fmt.Errorf("sriov-config-service failed to read OpenStack data: %v", err) + } + + openStackDevicesInfo, err := utils.CreateOpenstackDevicesInfo(metaData, networkData) + if err != nil { + glog.Errorf("failed to read OpenStack data: %v", err) + return fmt.Errorf("sriov-config-service failed to read OpenStack data: %v", err) + } + + ifaceStatuses, err = utils.DiscoverSriovDevicesVirtual(openStackDevicesInfo) + if err != nil { + glog.Errorf("sriov-config-service:failed to read OpenStack data: %v", err) + return fmt.Errorf("sriov-config-service: failed to read OpenStack data: %v", err) + } + + // Create the virtual plugin + configPlugin, err = virtual.NewVirtualPlugin(true) + if err != nil { + glog.Errorf("sriov-config-service: failed to create virtual plugin %v", err) + return fmt.Errorf("sriov-config-service: failed to create virtual plugin %v", err) + } + } + + nodeState := &sriovv1.SriovNetworkNodeState{ + Spec: nodeStateSpec.Spec, + Status: sriovv1.SriovNetworkNodeStateStatus{Interfaces: ifaceStatuses}, + } + + _, _, err = configPlugin.OnNodeStateChange(nodeState) + if err != nil { + glog.Errorf("sriov-config-service: failed to run OnNodeStateChange to update the generic plugin status %v", err) + return fmt.Errorf("sriov-config-service: failed to run OnNodeStateChange to update the generic plugin status %v", err) + } + + sriovResult := &systemd.SriovResult{ + SyncStatus: "Succeeded", + LastSyncError: "", + } + + err = configPlugin.Apply() + if err != nil { + glog.Errorf("sriov-config-service failed to run apply node configuration %v", err) + sriovResult.SyncStatus = "Failed" + sriovResult.LastSyncError = err.Error() + } + + err = systemd.WriteSriovResult(sriovResult) + if err != nil { + glog.Errorf("sriov-config-service failed to write sriov result file with content %v error: %v", *sriovResult, err) + return fmt.Errorf("sriov-config-service failed to write sriov result file with content %v error: %v", *sriovResult, err) + } + + glog.V(0).Info("Shutting down sriov-config-service") + return nil +} diff --git a/cmd/sriov-network-config-daemon/start.go b/cmd/sriov-network-config-daemon/start.go index 0ab621970..b9ee61ac5 100644 --- a/cmd/sriov-network-config-daemon/start.go +++ b/cmd/sriov-network-config-daemon/start.go @@ -1,3 +1,18 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package main import ( @@ -39,13 +54,15 @@ var ( startOpts struct { kubeconfig string nodeName string + systemd bool } ) func init() { rootCmd.AddCommand(startCmd) startCmd.PersistentFlags().StringVar(&startOpts.kubeconfig, "kubeconfig", "", "Kubeconfig file to access a remote cluster (testing only)") - startCmd.PersistentFlags().StringVar(&startOpts.nodeName, "node-name", "", "kubernetes node name daemon is managing.") + startCmd.PersistentFlags().StringVar(&startOpts.nodeName, "node-name", "", "kubernetes node name daemon is managing") + startCmd.PersistentFlags().BoolVar(&startOpts.systemd, "use-systemd-service", false, "use config daemon in systemd mode") } func runStartCmd(cmd *cobra.Command, args []string) { @@ -165,7 +182,7 @@ func runStartCmd(cmd *cobra.Command, args []string) { glog.V(0).Infof("Running on platform: %s", platformType.String()) var namespace = os.Getenv("NAMESPACE") - if err := sriovnetworkv1.InitNicIDMap(kubeclient, namespace); err != nil { + if err := sriovnetworkv1.InitNicIDMapFromConfigMap(kubeclient, namespace); err != nil { glog.Errorf("failed to run init NicIdMap: %v", err) panic(err.Error()) } @@ -189,6 +206,8 @@ func runStartCmd(cmd *cobra.Command, args []string) { syncCh, refreshCh, platformType, + startOpts.systemd, + devMode, ).Run(stopCh, exitCh) if err != nil { glog.Errorf("failed to run daemon: %v", err) diff --git a/pkg/consts/constants.go b/pkg/consts/constants.go index be19c4594..72e847d82 100644 --- a/pkg/consts/constants.go +++ b/pkg/consts/constants.go @@ -3,23 +3,25 @@ package consts import "time" const ( - ResyncPeriod = 5 * time.Minute - DefaultConfigName = "default" - ConfigDaemonPath = "./bindata/manifests/daemon" - InjectorWebHookPath = "./bindata/manifests/webhook" - OperatorWebHookPath = "./bindata/manifests/operator-webhook" - ServiceCAConfigMapAnnotation = "service.beta.openshift.io/inject-cabundle" - InjectorWebHookName = "network-resources-injector-config" - OperatorWebHookName = "sriov-operator-webhook-config" - DeprecatedOperatorWebHookName = "operator-webhook-config" - PluginPath = "./bindata/manifests/plugins" - DaemonPath = "./bindata/manifests/daemon" - DefaultPolicyName = "default" - ConfigMapName = "device-plugin-config" - DaemonSet = "DaemonSet" - ServiceAccount = "ServiceAccount" - DPConfigFileName = "config.json" - OVSHWOLMachineConfigNameSuffix = "ovs-hw-offload" + ResyncPeriod = 5 * time.Minute + DefaultConfigName = "default" + ConfigDaemonPath = "./bindata/manifests/daemon" + InjectorWebHookPath = "./bindata/manifests/webhook" + OperatorWebHookPath = "./bindata/manifests/operator-webhook" + SystemdServiceOcpPath = "./bindata/manifests/sriov-config-service/openshift" + SystemdServiceOcpMachineConfigName = "sriov-config-service" + ServiceCAConfigMapAnnotation = "service.beta.openshift.io/inject-cabundle" + InjectorWebHookName = "network-resources-injector-config" + OperatorWebHookName = "sriov-operator-webhook-config" + DeprecatedOperatorWebHookName = "operator-webhook-config" + PluginPath = "./bindata/manifests/plugins" + DaemonPath = "./bindata/manifests/daemon" + DefaultPolicyName = "default" + ConfigMapName = "device-plugin-config" + DaemonSet = "DaemonSet" + ServiceAccount = "ServiceAccount" + DPConfigFileName = "config.json" + OVSHWOLMachineConfigNameSuffix = "ovs-hw-offload" LinkTypeEthernet = "ether" LinkTypeInfiniband = "infiniband" diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 727ffe642..f257f8143 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -41,7 +41,10 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned" sninformer "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/informers/externalversions" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/service" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/systemd" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" ) @@ -65,6 +68,10 @@ type Daemon struct { platform utils.PlatformType + useSystemdService bool + + devMode bool + client snclientset.Interface // kubeClient allows interaction with Kubernetes, including the node we are running on. kubeClient kubernetes.Interface @@ -75,6 +82,8 @@ type Daemon struct { enabledPlugins map[string]plugin.VendorPlugin + serviceManager service.ServiceManager + // channel used by callbacks to signal Run() of an error exitCh chan<- error @@ -103,7 +112,6 @@ type Daemon struct { } const ( - rdmaScriptsPath = "/bindata/scripts/enable-rdma.sh" udevScriptsPath = "/bindata/scripts/load-udev.sh" annoKey = "sriovnetwork.openshift.io/state" annoIdle = "Idle" @@ -140,18 +148,23 @@ func New( syncCh <-chan struct{}, refreshCh chan<- Message, platformType utils.PlatformType, + useSystemdService bool, + devMode bool, ) *Daemon { return &Daemon{ - name: nodeName, - platform: platformType, - client: client, - kubeClient: kubeClient, - openshiftContext: openshiftContext, - exitCh: exitCh, - stopCh: stopCh, - syncCh: syncCh, - refreshCh: refreshCh, - nodeState: &sriovnetworkv1.SriovNetworkNodeState{}, + name: nodeName, + platform: platformType, + useSystemdService: useSystemdService, + devMode: devMode, + client: client, + kubeClient: kubeClient, + openshiftContext: openshiftContext, + serviceManager: service.NewServiceManager("/host"), + exitCh: exitCh, + stopCh: stopCh, + syncCh: syncCh, + refreshCh: refreshCh, + nodeState: &sriovnetworkv1.SriovNetworkNodeState{}, drainer: &drain.Helper{ Client: kubeClient, Force: true, @@ -210,13 +223,24 @@ func (dn *Daemon) Run(stopCh <-chan struct{}, exitCh <-chan error) error { } else { glog.V(0).Infof("Run(): start daemon.") } + + if dn.useSystemdService { + glog.V(0).Info("Run(): daemon running in systemd mode") + } // Only watch own SriovNetworkNodeState CR defer utilruntime.HandleCrash() defer dn.workqueue.ShutDown() - tryEnableRdma() - tryEnableTun() - tryEnableVhostNet() + if !dn.useSystemdService { + hostManager := host.NewHostManager(dn.useSystemdService) + hostManager.TryEnableRdma() + hostManager.TryEnableTun() + hostManager.TryEnableVhostNet() + err := systemd.CleanSriovFilesFromHost(utils.ClusterType == utils.ClusterTypeOpenshift) + if err != nil { + glog.Warningf("failed to remove all the systemd sriov files error: %v", err) + } + } if err := dn.tryCreateUdevRuleWrapper(); err != nil { return err @@ -274,7 +298,7 @@ func (dn *Daemon) Run(stopCh <-chan struct{}, exitCh <-chan error) error { } glog.Info("Starting workers") - // Launch one workers to process + // Launch one worker to process go wait.Until(dn.runWorker, time.Second, stopCh) glog.Info("Started workers") @@ -412,6 +436,7 @@ func (dn *Daemon) nodeStateSyncHandler() error { var err error // Get the latest NodeState var latestState *sriovnetworkv1.SriovNetworkNodeState + var sriovResult = &systemd.SriovResult{SyncStatus: syncStatusSucceeded, LastSyncError: ""} latestState, err = dn.client.SriovnetworkV1().SriovNetworkNodeStates(namespace).Get(context.Background(), dn.name, metav1.GetOptions{}) if err != nil { glog.Warningf("nodeStateSyncHandler(): Failed to fetch node state %s: %v", dn.name, err) @@ -420,7 +445,45 @@ func (dn *Daemon) nodeStateSyncHandler() error { latest := latestState.GetGeneration() glog.V(0).Infof("nodeStateSyncHandler(): new generation is %d", latest) + if utils.ClusterType == utils.ClusterTypeOpenshift && !dn.openshiftContext.IsHypershift() { + if err = dn.getNodeMachinePool(); err != nil { + return err + } + } + if dn.nodeState.GetGeneration() == latest { + if dn.useSystemdService { + serviceExist, err := dn.serviceManager.IsServiceExist(systemd.SriovServicePath) + if err != nil { + glog.Errorf("nodeStateSyncHandler(): failed to check if sriov-config service exist on host: %v", err) + return err + } + + // if the service doesn't exist we should continue to let the k8s plugin to create the service files + // this is only for k8s base environments, for openshift the sriov-operator creates a machine config to will apply + // the system service and reboot the node the config-daemon doesn't need to do anything. + if !serviceExist { + sriovResult = &systemd.SriovResult{SyncStatus: syncStatusFailed, LastSyncError: "sriov-config systemd service doesn't exist on node"} + } else { + sriovResult, err = systemd.ReadSriovResult() + if err != nil { + glog.Errorf("nodeStateSyncHandler(): failed to load sriov result file from host: %v", err) + return err + } + } + if sriovResult.LastSyncError != "" || sriovResult.SyncStatus == syncStatusFailed { + glog.Infof("nodeStateSyncHandler(): sync failed systemd service error: %s", sriovResult.LastSyncError) + + // add the error but don't requeue + dn.refreshCh <- Message{ + syncStatus: syncStatusFailed, + lastSyncError: sriovResult.LastSyncError, + } + <-dn.syncCh + return nil + } + return nil + } glog.V(0).Infof("nodeStateSyncHandler(): Interface not changed") if latestState.Status.LastSyncError != "" || latestState.Status.SyncStatus != syncStatusSucceeded { @@ -453,9 +516,9 @@ func (dn *Daemon) nodeStateSyncHandler() error { lastSyncError: "", } - // load plugins if has not loaded + // load plugins if it has not loaded if len(dn.enabledPlugins) == 0 { - dn.enabledPlugins, err = enablePlugins(dn.platform, latestState) + dn.enabledPlugins, err = enablePlugins(dn.platform, dn.useSystemdService, latestState) if err != nil { glog.Errorf("nodeStateSyncHandler(): failed to enable vendor plugins error: %v", err) return err @@ -464,6 +527,8 @@ func (dn *Daemon) nodeStateSyncHandler() error { reqReboot := false reqDrain := false + + // check if any of the plugins required to drain or reboot the node for k, p := range dn.enabledPlugins { d, r := false, false if dn.nodeState.GetName() == "" { @@ -480,10 +545,31 @@ func (dn *Daemon) nodeStateSyncHandler() error { reqDrain = reqDrain || d reqReboot = reqReboot || r } - glog.V(0).Infof("nodeStateSyncHandler(): reqDrain %v, reqReboot %v disableDrain %v", reqDrain, reqReboot, dn.disableDrain) + + // When running using systemd check if the applied configuration is the latest one + // or there is a new config we need to apply + // When using systemd configuration we write the file + if dn.useSystemdService { + r, err := systemd.WriteConfFile(latestState, dn.devMode, dn.platform) + if err != nil { + glog.Errorf("nodeStateSyncHandler(): failed to write configuration file for systemd mode: %v", err) + return err + } + reqDrain = reqDrain || r + reqReboot = reqReboot || r + glog.V(0).Infof("nodeStateSyncHandler(): systemd mode reqDrain %v, reqReboot %v disableDrain %v", r, r, dn.disableDrain) + + err = systemd.WriteSriovSupportedNics() + if err != nil { + glog.Errorf("nodeStateSyncHandler(): failed to write supported nic ids file for systemd mode: %v", err) + return err + } + } + glog.V(0).Infof("nodeStateSyncHandler(): aggregated daemon reqDrain %v, reqReboot %v disableDrain %v", reqDrain, reqReboot, dn.disableDrain) for k, p := range dn.enabledPlugins { - if k != GenericPluginName { + // Skip both the general and virtual plugin apply them last + if k != GenericPluginName && k != VirtualPluginName { err := p.Apply() if err != nil { glog.Errorf("nodeStateSyncHandler(): plugin %s fail to apply: %v", k, err) @@ -522,7 +608,8 @@ func (dn *Daemon) nodeStateSyncHandler() error { } } - if !reqReboot { + if !reqReboot && !dn.useSystemdService { + // For BareMetal machines apply the generic plugin selectedPlugin, ok := dn.enabledPlugins[GenericPluginName] if ok { // Apply generic_plugin last @@ -532,6 +619,17 @@ func (dn *Daemon) nodeStateSyncHandler() error { return err } } + + // For Virtual machines apply the virtual plugin + selectedPlugin, ok = dn.enabledPlugins[VirtualPluginName] + if ok { + // Apply virtual_plugin last + err = selectedPlugin.Apply() + if err != nil { + glog.Errorf("nodeStateSyncHandler(): generic_plugin fail to apply: %v", err) + return err + } + } } if reqReboot { @@ -561,9 +659,16 @@ func (dn *Daemon) nodeStateSyncHandler() error { } glog.Info("nodeStateSyncHandler(): sync succeeded") dn.nodeState = latestState.DeepCopy() - dn.refreshCh <- Message{ - syncStatus: syncStatusSucceeded, - lastSyncError: "", + if dn.useSystemdService { + dn.refreshCh <- Message{ + syncStatus: sriovResult.SyncStatus, + lastSyncError: sriovResult.LastSyncError, + } + } else { + dn.refreshCh <- Message{ + syncStatus: syncStatusSucceeded, + lastSyncError: "", + } } // wait for writer to refresh the status <-dn.syncCh @@ -936,44 +1041,6 @@ func (dn *Daemon) drainNode() error { return nil } -func tryEnableTun() { - if err := utils.LoadKernelModule("tun"); err != nil { - glog.Errorf("tryEnableTun(): TUN kernel module not loaded: %v", err) - } -} - -func tryEnableVhostNet() { - if err := utils.LoadKernelModule("vhost_net"); err != nil { - glog.Errorf("tryEnableVhostNet(): VHOST_NET kernel module not loaded: %v", err) - } -} - -func tryEnableRdma() (bool, error) { - glog.V(2).Infof("tryEnableRdma()") - var stdout, stderr bytes.Buffer - - cmd := exec.Command("/bin/bash", path.Join(filesystemRoot, rdmaScriptsPath)) - cmd.Stdout = &stdout - cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { - glog.Errorf("tryEnableRdma(): fail to enable rdma %v: %v", err, cmd.Stderr) - return false, err - } - glog.V(2).Infof("tryEnableRdma(): %v", cmd.Stdout) - - i, err := strconv.Atoi(strings.TrimSpace(stdout.String())) - if err == nil { - if i == 0 { - glog.V(2).Infof("tryEnableRdma(): RDMA kernel modules loaded") - return true, nil - } else { - glog.V(2).Infof("tryEnableRdma(): RDMA kernel modules not loaded") - return false, nil - } - } - return false, err -} - func tryCreateSwitchdevUdevRule(nodeState *sriovnetworkv1.SriovNetworkNodeState) error { glog.V(2).Infof("tryCreateSwitchdevUdevRule()") var newContent string diff --git a/pkg/daemon/daemon_test.go b/pkg/daemon/daemon_test.go index f6c5c4162..bf967af24 100644 --- a/pkg/daemon/daemon_test.go +++ b/pkg/daemon/daemon_test.go @@ -101,7 +101,7 @@ var _ = Describe("Config Daemon", func() { kubeClient := fakek8s.NewSimpleClientset(&FakeSupportedNicIDs, &SriovDevicePluginPod) client := fakesnclientset.NewSimpleClientset() - err = sriovnetworkv1.InitNicIDMap(kubeClient, namespace) + err = sriovnetworkv1.InitNicIDMapFromConfigMap(kubeClient, namespace) Expect(err).ToNot(HaveOccurred()) sut = New("test-node", @@ -113,6 +113,8 @@ var _ = Describe("Config Daemon", func() { syncCh, refreshCh, utils.Baremetal, + false, + false, ) sut.enabledPlugins = map[string]plugin.VendorPlugin{generic.PluginName: &fake.FakePlugin{}} diff --git a/pkg/daemon/plugin.go b/pkg/daemon/plugin.go index 307a35343..9639db88e 100644 --- a/pkg/daemon/plugin.go +++ b/pkg/daemon/plugin.go @@ -24,15 +24,16 @@ var ( GenericPlugin = genericplugin.NewGenericPlugin GenericPluginName = genericplugin.PluginName VirtualPlugin = virtualplugin.NewVirtualPlugin + VirtualPluginName = virtualplugin.PluginName K8sPlugin = k8splugin.NewK8sPlugin ) -func enablePlugins(platform utils.PlatformType, ns *sriovnetworkv1.SriovNetworkNodeState) (map[string]plugin.VendorPlugin, error) { +func enablePlugins(platform utils.PlatformType, useSystemdService bool, ns *sriovnetworkv1.SriovNetworkNodeState) (map[string]plugin.VendorPlugin, error) { glog.Infof("enableVendorPlugins(): enabling plugins") enabledPlugins := map[string]plugin.VendorPlugin{} if platform == utils.VirtualOpenStack { - virtualPlugin, err := VirtualPlugin() + virtualPlugin, err := VirtualPlugin(false) if err != nil { glog.Errorf("enableVendorPlugins(): failed to load the virtual plugin error: %v", err) return nil, err @@ -46,14 +47,14 @@ func enablePlugins(platform utils.PlatformType, ns *sriovnetworkv1.SriovNetworkN enabledPlugins = enabledVendorPlugins if utils.ClusterType != utils.ClusterTypeOpenshift { - k8sPlugin, err := K8sPlugin() + k8sPlugin, err := K8sPlugin(useSystemdService) if err != nil { glog.Errorf("enableVendorPlugins(): failed to load the k8s plugin error: %v", err) return nil, err } enabledPlugins[k8sPlugin.Name()] = k8sPlugin } - genericPlugin, err := GenericPlugin() + genericPlugin, err := GenericPlugin(false) if err != nil { glog.Errorf("enableVendorPlugins(): failed to load the generic plugin error: %v", err) return nil, err diff --git a/pkg/daemon/writer.go b/pkg/daemon/writer.go index d70ea8843..2a6fc6384 100644 --- a/pkg/daemon/writer.go +++ b/pkg/daemon/writer.go @@ -54,7 +54,7 @@ func (w *NodeStateStatusWriter) RunOnce(destDir string, platformType utils.Platf } if ns == nil { - metaData, networkData, err := utils.GetOpenstackData() + metaData, networkData, err := utils.GetOpenstackData(true) if err != nil { glog.Errorf("RunOnce(): failed to read OpenStack data: %v", err) } @@ -109,10 +109,7 @@ func (w *NodeStateStatusWriter) Run(stop <-chan struct{}, refresh <-chan Message if err := w.pollNicStatus(platformType); err != nil { continue } - _, err := w.setNodeStateStatus(msg) - if err != nil { - glog.Errorf("Run() period: writing to node status failed: %v", err) - } + w.setNodeStateStatus(msg) } } } diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go index a2a7e1c00..c5b8c3596 100644 --- a/pkg/plugins/generic/generic_plugin.go +++ b/pkg/plugins/generic/generic_plugin.go @@ -12,6 +12,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" ) @@ -25,6 +26,8 @@ type GenericPlugin struct { LastState *sriovnetworkv1.SriovNetworkNodeState LoadVfioDriver uint LoadVirtioVdpaDriver uint + RunningOnHost bool + HostManager host.HostManagerInterface } const scriptsPath = "bindata/scripts/enable-kargs.sh" @@ -36,12 +39,14 @@ const ( ) // Initialize our plugin and set up initial values -func NewGenericPlugin() (plugin.VendorPlugin, error) { +func NewGenericPlugin(runningOnHost bool) (plugin.VendorPlugin, error) { return &GenericPlugin{ PluginName: PluginName, SpecVersion: "1.0", LoadVfioDriver: unloaded, LoadVirtioVdpaDriver: unloaded, + RunningOnHost: runningOnHost, + HostManager: host.NewHostManager(runningOnHost), }, nil } @@ -76,7 +81,7 @@ func (p *GenericPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeSt func (p *GenericPlugin) Apply() error { glog.Infof("generic-plugin Apply(): desiredState=%v", p.DesireState.Spec) if p.LoadVfioDriver == loading { - if err := utils.LoadKernelModule("vfio_pci"); err != nil { + if err := p.HostManager.LoadKernelModule("vfio_pci"); err != nil { glog.Errorf("generic-plugin Apply(): fail to load vfio_pci kmod: %v", err) return err } @@ -84,7 +89,7 @@ func (p *GenericPlugin) Apply() error { } if p.LoadVirtioVdpaDriver == loading { - if err := utils.LoadKernelModule("virtio_vdpa"); err != nil { + if err := p.HostManager.LoadKernelModule("virtio_vdpa"); err != nil { glog.Errorf("generic-plugin Apply(): fail to load virtio_vdpa kmod: %v", err) return err } @@ -107,11 +112,15 @@ func (p *GenericPlugin) Apply() error { return err } - exit, err := utils.Chroot("/host") - if err != nil { - return err + // When calling from systemd do not try to chroot + if !p.RunningOnHost { + exit, err := utils.Chroot("/host") + if err != nil { + return err + } + defer exit() } - defer exit() + if err := utils.SyncNodeState(p.DesireState, pfsToSkip); err != nil { return err } diff --git a/pkg/plugins/generic/generic_plugin_test.go b/pkg/plugins/generic/generic_plugin_test.go index 69e850154..0d8fc0000 100644 --- a/pkg/plugins/generic/generic_plugin_test.go +++ b/pkg/plugins/generic/generic_plugin_test.go @@ -20,7 +20,7 @@ var _ = Describe("Generic plugin", func() { var genericPlugin plugin.VendorPlugin var err error BeforeEach(func() { - genericPlugin, err = generic.NewGenericPlugin() + genericPlugin, err = generic.NewGenericPlugin(false) Expect(err).ToNot(HaveOccurred()) }) diff --git a/pkg/plugins/k8s/k8s_plugin.go b/pkg/plugins/k8s/k8s_plugin.go index 948e51a5e..bd10408ff 100644 --- a/pkg/plugins/k8s/k8s_plugin.go +++ b/pkg/plugins/k8s/k8s_plugin.go @@ -29,7 +29,9 @@ type K8sPlugin struct { switchdevAfterNMService *service.Service openVSwitchService *service.Service networkManagerService *service.Service + sriovService *service.Service updateTarget *k8sUpdateTarget + useSystemdService bool } type k8sUpdateTarget struct { @@ -38,15 +40,16 @@ type k8sUpdateTarget struct { switchdevBeforeNMRunScript bool switchdevAfterNMRunScript bool switchdevUdevScript bool + sriovScript bool systemServices []*service.Service } func (u *k8sUpdateTarget) needUpdate() bool { - return u.switchdevBeforeNMService || u.switchdevAfterNMService || u.switchdevBeforeNMRunScript || u.switchdevAfterNMRunScript || u.switchdevUdevScript || len(u.systemServices) > 0 + return u.switchdevBeforeNMService || u.switchdevAfterNMService || u.switchdevBeforeNMRunScript || u.switchdevAfterNMRunScript || u.switchdevUdevScript || u.sriovScript || len(u.systemServices) > 0 } func (u *k8sUpdateTarget) needReboot() bool { - return u.switchdevBeforeNMService || u.switchdevAfterNMService || u.switchdevBeforeNMRunScript || u.switchdevAfterNMRunScript || u.switchdevUdevScript + return u.switchdevBeforeNMService || u.switchdevAfterNMService || u.switchdevBeforeNMRunScript || u.switchdevAfterNMRunScript || u.switchdevUdevScript || u.sriovScript } func (u *k8sUpdateTarget) reset() { @@ -54,6 +57,8 @@ func (u *k8sUpdateTarget) reset() { u.switchdevAfterNMService = false u.switchdevBeforeNMRunScript = false u.switchdevAfterNMRunScript = false + u.switchdevUdevScript = false + u.sriovScript = false u.systemServices = []*service.Service{} } @@ -76,8 +81,11 @@ func (u *k8sUpdateTarget) String() string { } const ( - switchdevManifestPath = "bindata/manifests/switchdev-config/" + bindataManifestPath = "bindata/manifests/" + switchdevManifestPath = bindataManifestPath + "switchdev-config/" switchdevUnits = switchdevManifestPath + "switchdev-units/" + sriovUnits = bindataManifestPath + "sriov-config-service/kubernetes/" + sriovUnitFile = sriovUnits + "sriov-config-service.yaml" switchdevBeforeNMUnitFile = switchdevUnits + "switchdev-configuration-before-nm.yaml" switchdevAfterNMUnitFile = switchdevUnits + "switchdev-configuration-after-nm.yaml" networkManagerUnitFile = switchdevUnits + "NetworkManager.service.yaml" @@ -90,12 +98,13 @@ const ( ) // Initialize our plugin and set up initial values -func NewK8sPlugin() (plugins.VendorPlugin, error) { +func NewK8sPlugin(useSystemdService bool) (plugins.VendorPlugin, error) { k8sPluging := &K8sPlugin{ - PluginName: PluginName, - SpecVersion: "1.0", - serviceManager: service.NewServiceManager(chroot), - updateTarget: &k8sUpdateTarget{}, + PluginName: PluginName, + SpecVersion: "1.0", + serviceManager: service.NewServiceManager(chroot), + updateTarget: &k8sUpdateTarget{}, + useSystemdService: useSystemdService, } return k8sPluging, k8sPluging.readManifestFiles() @@ -120,15 +129,26 @@ func (p *K8sPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) p.updateTarget.reset() // TODO add check for enableOvsOffload in OperatorConfig later // Update services if switchdev required - if !utils.IsSwitchdevModeSpec(new.Spec) { + if !p.useSystemdService && !utils.IsSwitchdevModeSpec(new.Spec) { return } - // Check services - err = p.servicesStateUpdate() - if err != nil { - glog.Errorf("k8s-plugin OnNodeStateChange(): failed : %v", err) - return + if utils.IsSwitchdevModeSpec(new.Spec) { + // Check services + err = p.switchDevServicesStateUpdate() + if err != nil { + glog.Errorf("k8s-plugin OnNodeStateChange(): failed : %v", err) + return + } + } + + if p.useSystemdService { + // Check sriov service + err = p.sriovServiceStateUpdate() + if err != nil { + glog.Errorf("k8s-plugin OnNodeStateChange(): failed : %v", err) + return + } } if p.updateTarget.needUpdate() { @@ -151,6 +171,12 @@ func (p *K8sPlugin) Apply() error { return err } + if p.useSystemdService { + if err := p.updateSriovService(); err != nil { + return err + } + } + for _, systemService := range p.updateTarget.systemServices { if err := p.updateSystemService(systemService); err != nil { return err @@ -230,6 +256,16 @@ func (p *K8sPlugin) readOpenVSwitchdManifest() error { return nil } +func (p *K8sPlugin) readSriovServiceManifest() error { + sriovService, err := service.ReadServiceManifestFile(sriovUnitFile) + if err != nil { + return err + } + + p.sriovService = sriovService + return nil +} + func (p *K8sPlugin) readManifestFiles() error { if err := p.readSwitchdevManifest(); err != nil { return err @@ -243,6 +279,10 @@ func (p *K8sPlugin) readManifestFiles() error { return err } + if err := p.readSriovServiceManifest(); err != nil { + return err + } + return nil } @@ -281,7 +321,27 @@ func (p *K8sPlugin) switchdevServiceStateUpdate() error { return nil } -func (p *K8sPlugin) getSystemServices() []*service.Service { +func (p *K8sPlugin) sriovServiceStateUpdate() error { + glog.Info("sriovServiceStateUpdate()") + exist, err := p.serviceManager.IsServiceExist(p.sriovService.Path) + if err != nil { + return err + } + + // create the service if it doesn't exist + if !exist { + p.updateTarget.sriovScript = true + } else { + p.updateTarget.sriovScript = p.isSystemServiceNeedUpdate(p.sriovService) + } + + if p.updateTarget.sriovScript { + p.updateTarget.systemServices = append(p.updateTarget.systemServices, p.sriovService) + } + return nil +} + +func (p *K8sPlugin) getSwitchDevSystemServices() []*service.Service { return []*service.Service{p.networkManagerService, p.openVSwitchService} } @@ -316,16 +376,17 @@ func (p *K8sPlugin) isSwitchdevServiceNeedUpdate(serviceObj *service.Service) (n } func (p *K8sPlugin) isSystemServiceNeedUpdate(serviceObj *service.Service) bool { + glog.Infof("isSystemServiceNeedUpdate()") systemService, err := p.serviceManager.ReadService(serviceObj.Path) if err != nil { - glog.Warningf("k8s-plugin isSystemServiceNeedUpdate(): failed to read switchdev service file %q: %v", + glog.Warningf("k8s-plugin isSystemServiceNeedUpdate(): failed to read sriov-config service file %q: %v", serviceObj.Path, err) return false } if systemService != nil { needChange, err := service.CompareServices(systemService, serviceObj) if err != nil { - glog.Warningf("k8s-plugin isSystemServiceNeedUpdate(): failed to compare switchdev service : %v", err) + glog.Warningf("k8s-plugin isSystemServiceNeedUpdate(): failed to compare sriov-config service: %v", err) return false } return needChange @@ -336,7 +397,7 @@ func (p *K8sPlugin) isSystemServiceNeedUpdate(serviceObj *service.Service) bool func (p *K8sPlugin) systemServicesStateUpdate() error { var services []*service.Service - for _, systemService := range p.getSystemServices() { + for _, systemService := range p.getSwitchDevSystemServices() { exist, err := p.serviceManager.IsServiceExist(systemService.Path) if err != nil { return err @@ -353,7 +414,7 @@ func (p *K8sPlugin) systemServicesStateUpdate() error { return nil } -func (p *K8sPlugin) servicesStateUpdate() error { +func (p *K8sPlugin) switchDevServicesStateUpdate() error { // Check switchdev err := p.switchdevServiceStateUpdate() if err != nil { @@ -369,6 +430,17 @@ func (p *K8sPlugin) servicesStateUpdate() error { return nil } +func (p *K8sPlugin) updateSriovService() error { + if p.updateTarget.sriovScript { + err := p.serviceManager.EnableService(p.sriovService) + if err != nil { + return err + } + } + + return nil +} + func (p *K8sPlugin) updateSwitchdevService() error { if p.updateTarget.switchdevBeforeNMService { err := p.serviceManager.EnableService(p.switchdevBeforeNMService) @@ -418,7 +490,7 @@ func (p *K8sPlugin) updateSystemService(serviceObj *service.Service) error { } if systemService == nil { // Invalid case to reach here - return fmt.Errorf("k8s-plugin Apply(): can't update non-existing service %q", serviceObj.Name) + return fmt.Errorf("k8s-plugin updateSystemService(): can't update non-existing service %q", serviceObj.Name) } serviceOptions, err := unit.Deserialize(strings.NewReader(serviceObj.Content)) if err != nil { @@ -431,3 +503,11 @@ func (p *K8sPlugin) updateSystemService(serviceObj *service.Service) error { return p.serviceManager.EnableService(updatedService) } + +func (p *K8sPlugin) SetSystemdFlag() { + p.useSystemdService = true +} + +func (p *K8sPlugin) IsSystemService() bool { + return p.useSystemdService +} diff --git a/pkg/plugins/virtual/virtual_plugin.go b/pkg/plugins/virtual/virtual_plugin.go index 2d74cbbd3..dbe0d9883 100644 --- a/pkg/plugins/virtual/virtual_plugin.go +++ b/pkg/plugins/virtual/virtual_plugin.go @@ -7,6 +7,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" ) @@ -20,6 +21,8 @@ type VirtualPlugin struct { DesireState *sriovnetworkv1.SriovNetworkNodeState LastState *sriovnetworkv1.SriovNetworkNodeState LoadVfioDriver uint + RunningOnHost bool + HostManager host.HostManagerInterface } const ( @@ -29,11 +32,13 @@ const ( ) // Initialize our plugin and set up initial values -func NewVirtualPlugin() (plugin.VendorPlugin, error) { +func NewVirtualPlugin(runningOnHost bool) (plugin.VendorPlugin, error) { return &VirtualPlugin{ PluginName: PluginName, SpecVersion: "1.0", LoadVfioDriver: unloaded, + RunningOnHost: runningOnHost, + HostManager: host.NewHostManager(runningOnHost), }, nil } @@ -74,12 +79,12 @@ func (p *VirtualPlugin) Apply() error { // This is the case for OpenStack deployments where the underlying virtualization platform is KVM. // NOTE: if VFIO was already loaded for some reason, we will not try to load it again with the new options. kernelArgs := "enable_unsafe_noiommu_mode=1" - if err := utils.LoadKernelModule("vfio", kernelArgs); err != nil { + if err := p.HostManager.LoadKernelModule("vfio", kernelArgs); err != nil { glog.Errorf("virtual-plugin Apply(): fail to load vfio kmod: %v", err) return err } - if err := utils.LoadKernelModule("vfio_pci"); err != nil { + if err := p.HostManager.LoadKernelModule("vfio_pci"); err != nil { glog.Errorf("virtual-plugin Apply(): fail to load vfio_pci kmod: %v", err) return err } @@ -107,6 +112,13 @@ func (p *VirtualPlugin) Apply() error { return nil } +func (p *VirtualPlugin) SetSystemdFlag() { +} + +func (p *VirtualPlugin) IsSystemService() bool { + return false +} + func needVfioDriver(state *sriovnetworkv1.SriovNetworkNodeState) bool { for _, iface := range state.Spec.Interfaces { for i := range iface.VfGroups { diff --git a/pkg/service/utils.go b/pkg/service/utils.go index 8b9e82c72..f9dc28f85 100644 --- a/pkg/service/utils.go +++ b/pkg/service/utils.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/coreos/go-systemd/v22/unit" + "github.com/golang/glog" "gopkg.in/yaml.v2" ) @@ -28,7 +29,7 @@ OUTER: continue OUTER } } - + glog.Infof("DEBUG: %+v %v", optsA, *optB) return true, nil } diff --git a/pkg/systemd/systemd.go b/pkg/systemd/systemd.go new file mode 100644 index 000000000..127b897be --- /dev/null +++ b/pkg/systemd/systemd.go @@ -0,0 +1,288 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package systemd + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/golang/glog" + "gopkg.in/yaml.v3" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" +) + +const ( + SriovSystemdConfigPath = utils.SriovConfBasePath + "/sriov-interface-config.yaml" + SriovSystemdResultPath = utils.SriovConfBasePath + "/sriov-interface-result.yaml" + sriovSystemdSupportedNicPath = utils.SriovConfBasePath + "/sriov-supported-nics-ids.yaml" + sriovSystemdServiceBinaryPath = "/var/lib/sriov/sriov-network-config-daemon" + + SriovHostSystemdConfigPath = "/host" + SriovSystemdConfigPath + SriovHostSystemdResultPath = "/host" + SriovSystemdResultPath + sriovHostSystemdSupportedNicPath = "/host" + sriovSystemdSupportedNicPath + sriovHostSystemdServiceBinaryPath = "/host" + sriovSystemdServiceBinaryPath + + SriovServicePath = "/etc/systemd/system/sriov-config.service" + SriovHostServicePath = "/host" + SriovServicePath +) + +type SriovConfig struct { + Spec sriovnetworkv1.SriovNetworkNodeStateSpec `yaml:"spec"` + UnsupportedNics bool `yaml:"unsupportedNics"` + PlatformType utils.PlatformType `yaml:"platformType"` +} + +type SriovResult struct { + SyncStatus string `yaml:"syncStatus"` + LastSyncError string `yaml:"lastSyncError"` +} + +func ReadConfFile() (spec *SriovConfig, err error) { + rawConfig, err := ioutil.ReadFile(SriovSystemdConfigPath) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(rawConfig, &spec) + + return spec, err +} + +func WriteConfFile(newState *sriovnetworkv1.SriovNetworkNodeState, unsupportedNics bool, platformType utils.PlatformType) (bool, error) { + newFile := false + // remove the device plugin revision as we don't need it here + newState.Spec.DpConfigVersion = "" + + sriovConfig := &SriovConfig{ + newState.Spec, + unsupportedNics, + platformType, + } + + _, err := os.Stat(SriovHostSystemdConfigPath) + if err != nil { + if os.IsNotExist(err) { + // Create the sriov-operator folder on the host if it doesn't exist + if _, err := os.Stat(utils.HostSriovConfBasePath); os.IsNotExist(err) { + err = os.Mkdir(utils.HostSriovConfBasePath, os.ModeDir) + if err != nil { + glog.Errorf("WriteConfFile(): fail to create sriov-operator folder: %v", err) + return false, err + } + } + + glog.V(2).Infof("WriteConfFile(): file not existed, create it") + _, err = os.Create(SriovHostSystemdConfigPath) + if err != nil { + glog.Errorf("WriteConfFile(): fail to create file: %v", err) + return false, err + } + newFile = true + } else { + return false, err + } + } + + oldContent, err := ioutil.ReadFile(SriovHostSystemdConfigPath) + if err != nil { + glog.Errorf("WriteConfFile(): fail to read file: %v", err) + return false, err + } + + oldContentObj := &SriovConfig{} + err = yaml.Unmarshal(oldContent, oldContentObj) + if err != nil { + glog.Errorf("WriteConfFile(): fail to unmarshal old file: %v", err) + return false, err + } + + var newContent []byte + newContent, err = yaml.Marshal(sriovConfig) + if err != nil { + glog.Errorf("WriteConfFile(): fail to marshal config: %v", err) + return false, err + } + + if bytes.Equal(newContent, oldContent) { + glog.V(2).Info("WriteConfFile(): no update") + return false, nil + } + glog.V(2).Infof("WriteConfFile(): previews configuration is not equal: old config:\n%s\nnew config:\n%s\n", string(oldContent), string(newContent)) + + glog.V(2).Infof("WriteConfFile(): write '%s' to %s", newContent, SriovHostSystemdConfigPath) + err = ioutil.WriteFile(SriovHostSystemdConfigPath, newContent, 0644) + if err != nil { + glog.Errorf("WriteConfFile(): fail to write file: %v", err) + return false, err + } + + // this will be used to mark the first time we create this file. + // this helps to avoid the first reboot after installation + if newFile && len(sriovConfig.Spec.Interfaces) == 0 { + glog.V(2).Info("WriteConfFile(): first file creation and no interfaces to configure returning reboot false") + return false, nil + } + + return true, nil +} + +func WriteSriovResult(result *SriovResult) error { + _, err := os.Stat(SriovSystemdResultPath) + if err != nil { + if os.IsNotExist(err) { + glog.V(2).Infof("WriteSriovResult(): file not existed, create it") + _, err = os.Create(SriovSystemdResultPath) + if err != nil { + glog.Errorf("WriteSriovResult(): failed to create sriov result file on path %s: %v", SriovSystemdResultPath, err) + return err + } + } else { + glog.Errorf("WriteSriovResult(): failed to check sriov result file on path %s: %v", SriovSystemdResultPath, err) + return err + } + } + + out, err := yaml.Marshal(result) + if err != nil { + glog.Errorf("WriteSriovResult(): failed to marshal sriov result file: %v", err) + return err + } + + glog.V(2).Infof("WriteSriovResult(): write '%s' to %s", string(out), SriovSystemdResultPath) + err = ioutil.WriteFile(SriovSystemdResultPath, out, 0644) + if err != nil { + glog.Errorf("WriteSriovResult(): failed to write sriov result file on path %s: %v", SriovSystemdResultPath, err) + return err + } + + return nil +} + +func ReadSriovResult() (*SriovResult, error) { + _, err := os.Stat(SriovHostSystemdResultPath) + if err != nil { + if os.IsNotExist(err) { + glog.V(2).Infof("ReadSriovResult(): file not existed, return empty result") + return &SriovResult{}, nil + } else { + glog.Errorf("ReadSriovResult(): failed to check sriov result file on path %s: %v", SriovHostSystemdResultPath, err) + return nil, err + } + } + + rawConfig, err := ioutil.ReadFile(SriovHostSystemdResultPath) + if err != nil { + glog.Errorf("ReadSriovResult(): failed to read sriov result file on path %s: %v", SriovHostSystemdResultPath, err) + return nil, err + } + + result := &SriovResult{} + err = yaml.Unmarshal(rawConfig, &result) + if err != nil { + glog.Errorf("ReadSriovResult(): failed to unmarshal sriov result file on path %s: %v", SriovHostSystemdResultPath, err) + return nil, err + } + return result, err +} + +func WriteSriovSupportedNics() error { + _, err := os.Stat(sriovHostSystemdSupportedNicPath) + if err != nil { + if os.IsNotExist(err) { + glog.V(2).Infof("WriteSriovSupportedNics(): file not existed, create it") + _, err = os.Create(sriovHostSystemdSupportedNicPath) + if err != nil { + glog.Errorf("WriteSriovSupportedNics(): failed to create sriov supporter nics ids file on path %s: %v", sriovHostSystemdSupportedNicPath, err) + return err + } + } else { + glog.Errorf("WriteSriovSupportedNics(): failed to check sriov supporter nics ids file on path %s: %v", sriovHostSystemdSupportedNicPath, err) + return err + } + } + + rawNicList := []byte{} + for _, line := range sriovnetworkv1.NicIDMap { + rawNicList = append(rawNicList, []byte(fmt.Sprintf("%s\n", line))...) + } + + err = ioutil.WriteFile(sriovHostSystemdSupportedNicPath, rawNicList, 0644) + if err != nil { + glog.Errorf("WriteSriovSupportedNics(): failed to write sriov supporter nics ids file on path %s: %v", sriovHostSystemdSupportedNicPath, err) + return err + } + + return nil +} + +func ReadSriovSupportedNics() ([]string, error) { + _, err := os.Stat(sriovSystemdSupportedNicPath) + if err != nil { + if os.IsNotExist(err) { + glog.V(2).Infof("ReadSriovSupportedNics(): file not existed, return empty result") + return nil, err + } else { + glog.Errorf("ReadSriovSupportedNics(): failed to check sriov supporter nics file on path %s: %v", sriovSystemdSupportedNicPath, err) + return nil, err + } + } + + rawConfig, err := ioutil.ReadFile(sriovSystemdSupportedNicPath) + if err != nil { + glog.Errorf("ReadSriovSupportedNics(): failed to read sriov supporter nics file on path %s: %v", sriovSystemdSupportedNicPath, err) + return nil, err + } + + lines := strings.Split(string(rawConfig), "\n") + return lines, nil +} + +func CleanSriovFilesFromHost(isOpenShift bool) error { + err := os.Remove(SriovHostSystemdConfigPath) + if err != nil && !os.IsNotExist(err) { + return err + } + + err = os.Remove(SriovHostSystemdResultPath) + if err != nil && !os.IsNotExist(err) { + return err + } + + err = os.Remove(sriovHostSystemdSupportedNicPath) + if err != nil && !os.IsNotExist(err) { + return err + } + + err = os.Remove(sriovHostSystemdServiceBinaryPath) + if err != nil && !os.IsNotExist(err) { + return err + } + + // in openshift we should not remove the systemd service it will be done by the machine config operator + if !isOpenShift { + err = os.Remove(SriovHostServicePath) + if err != nil && !os.IsNotExist(err) { + return err + } + } + + return nil +}