Implement dedicated services, enhance integration tests, ensure ranch…

…er-wins can restart in all scenarios

cmd/server/app/run.go

@@ -41,6 +41,11 @@ var _runFlags = []cli.Flag{
 		Usage: "[optional] Specifies the name of the file to write the profile to",
 		Value: "profile.pprof",
 	},
+	&cli.BoolFlag{
+		Name:  "delayed-start",
+		Usage: "[optional] configure the rancher-wins service with a start type of 'Automatic (Delayed)'",
+		Value: false,
+	},
 }
 
 func _profilingInit(cliCtx *cli.Context) error {

suc/pkg/cmd.go

@@ -42,7 +42,7 @@ func Run(_ *cli.Context) error {
 
 	// Neither changing the start type nor service
 	// dependencies require any service restarts
-	err = service.ConfigureDelayedStart()
+	err = service.ConfigureWinsDelayedStart()
 	if err != nil {
 		errs = append(errs, err)
 	}
@@ -63,10 +63,11 @@ func Run(_ *cli.Context) error {
 		logrus.Errorf("Attempting to restore initial state due to error(s) encountered while updating rancher-wins: %v", errors.Join(errs...))
 		err = state.RestoreInitialState(initialState)
 		if err != nil {
-			return fmt.Errorf("failed to restore initial state: %w", err)
+			errs = append(errs, fmt.Errorf("failed to restore initial state: %w", err))
+		} else {
+			logrus.Info("Successfully restored initial config state")
 		}
-		logrus.Info("Successfully restored initial config state")
-		return fmt.Errorf("failed to update rancher-wins config file: %w", updateErr)
+		return errors.Join(errs...)
 	}
 
 	return nil

suc/pkg/service/common.go

@@ -5,6 +5,9 @@ import (
 )
 
 func SlicesMatch(s1 []string, s2 []string) bool {
+	if len(s1) != len(s2) {
+		return false
+	}
 	for _, e := range s1 {
 		found := false
 		for _, e2 := range s2 {

suc/pkg/service/config/config.go

@@ -77,7 +77,7 @@ func UpdateConfigFromEnvVars() (bool, error) {
 	logrus.Infof("Checking the %s value. This is a boolean flag, expecting 'true' or 'false'", AgentStringTLSEnvVar)
 	if v := os.Getenv(AgentStringTLSEnvVar); v != "" {
 		logrus.Infof("Found value '%s' for %s", v, AgentStringTLSEnvVar)
-		givenBool := strings.ToLower(v) == "true"
+		givenBool = strings.ToLower(v) == "true"
 		if cfg.AgentStrictTLSMode != givenBool {
 			cfg.AgentStrictTLSMode = givenBool
 			configNeedsUpdate = true

suc/pkg/service/configure.go

@@ -9,11 +9,16 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// ConfigureRKE2ServiceDependency creates a service dependency between rke2 and rancher-wins. This results in
+// rancher-wins becoming a dependant service for rke2, preventing rke2 startup until rancher-wins is ready. This
+// ensures that rancher-wins and rke2 do not interfere one another during start up (For example, due to CNI reconfiguration).
+// As a side effect, the rancher-wins service cannot be stopped if rke2 is still running. To restart rancher-wins,
+// this dependency must be temporarily removed.
 func ConfigureRKE2ServiceDependency() error {
 	logrus.Info("Configuring rke2 service dependencies")
 	add := strings.ToLower(os.Getenv("CATTLE_ENABLE_WINS_SERVICE_DEPENDENCY")) == "true"
 
-	rke2, serviceExists, err := Open("rke2")
+	rke2, serviceExists, err := OpenRke2Service()
 	if err != nil {
 		return fmt.Errorf("failed to open rke2 service while configuring service dependencies: %w", err)
 	}
@@ -24,58 +29,42 @@ func ConfigureRKE2ServiceDependency() error {
 	}
 	defer rke2.Close()
 
-	found := false
-	for _, dep := range rke2.Config.Dependencies {
-		logrus.Debugf("Found rke2 service dependency '%s'", dep)
-		if dep == defaults.WindowsServiceName {
-			logrus.Debugf("Found rancher-wins dependency set on rke2 service")
-			found = true
-			break
-		}
+	found, err := rke2.HasRancherWinsServiceDependency()
+	if err != nil {
+		return fmt.Errorf("error encountered determining rke2 service dependencies: %w", err)
 	}
 
-	saveChanges := false
 	if found && add {
-		logrus.Info("rke2 service dependency already configured")
+		logrus.Info("rke2 service dependency already configured, nothing to do")
 		return nil
 	}
 
 	if !found && add {
 		logrus.Info("Adding rancher-wins dependency on rke2 service")
-		saveChanges = true
-		rke2.Config.Dependencies = append(rke2.Config.Dependencies, defaults.WindowsServiceName)
+		err = rke2.AddRancherWinsServiceDependency()
+		if err != nil {
+			return fmt.Errorf("error encountered adding rke2 service dependency: %w", err)
+		}
 	}
 
 	if found && !add {
 		logrus.Info("Removing rancher-wins dependency on rke2 service")
-		saveChanges = true
-		rke2.Config.Dependencies = removeFromSlice(defaults.WindowsServiceName, rke2.Config.Dependencies)
-		if len(rke2.Config.Dependencies) == 0 {
-			// Updating a service config with a nil or empty Dependencies slice will not have any effect.
-			// Instead, '/' must be used to clear any remaining service dependencies.
-			rke2.Config.Dependencies = append(rke2.Config.Dependencies, "/")
+		err = rke2.RemoveRancherWinsServiceDependency()
+		if err != nil {
+			return fmt.Errorf("error encountered adding rke2 service dependency: %w", err)
 		}
 	}
 
-	if !saveChanges {
-		logrus.Info("No modification to rke2 service dependencies required")
-		return nil
-	}
-
-	logrus.Info("Updating rke2 service dependencies")
-	err = rke2.UpdateConfig()
-	if err != nil {
-		return fmt.Errorf("failed to update rke2 config while creating service dependencies: %w", err)
-	}
-
 	return nil
 }
 
-func ConfigureDelayedStart() error {
+// ConfigureWinsDelayedStart opens the rancher-wins service and enables the `DelayedAutoStart` flag.
+// Enabling this flag does not require a restart of the service.
+func ConfigureWinsDelayedStart() error {
 	logrus.Info("Configuring start type for rancher-wins")
 	delayedStart := strings.ToLower(os.Getenv("CATTLE_ENABLE_WINS_DELAYED_START")) == "true"
 
-	wins, exists, err := Open(defaults.WindowsServiceName)
+	wins, exists, err := OpenRancherWinsService()
 	if err != nil {
 		return fmt.Errorf("failed to open %s service while configuring start type: %w", defaults.WindowsServiceName, err)
 	}
@@ -87,26 +76,19 @@ func ConfigureDelayedStart() error {
 
 	defer wins.Close()
 
-	logrus.Infof("%s service has delayed auto start configured: %t", defaults.WindowsServiceName, wins.Config.DelayedAutoStart)
-
-	if wins.Config.DelayedAutoStart != delayedStart {
-		logrus.Infof("updating %s delayed auto start setting to %t", defaults.WindowsServiceName, delayedStart)
-		wins.Config.DelayedAutoStart = delayedStart
-		err = wins.UpdateConfig()
-		if err != nil {
-			return fmt.Errorf("failed to update %s service configuration while configuring service start type: %w", defaults.WindowsServiceName, err)
-		}
-		return nil
+	err = wins.ConfigureDelayedStart(delayedStart)
+	if err != nil {
+		return fmt.Errorf("error encountered configuring delayed start for %s service: %w", defaults.WindowsServiceName, err)
 	}
 
-	logrus.Infof("%s delayed start already set to %t", defaults.WindowsServiceName, delayedStart)
-
 	return nil
 }
 
+// RefreshWinsService restarts the rancher-wins service. If a service dependency has
+// been configured on the rke2 service, the dependency will be temporarily removed and
+// restored once the service restart has completed.
 func RefreshWinsService() error {
-	logrus.Infof("Restarting the %s service", defaults.WindowsServiceDisplayName)
-	winSrv, exists, err := Open(defaults.WindowsServiceName)
+	winSrv, exists, err := OpenRancherWinsService()
 	if err != nil {
 		logrus.Errorf("Cannot restart %s as the service failed to open: %v", defaults.WindowsServiceName, err)
 		return fmt.Errorf("failed to refresh the %s service: %w", winSrv.Name, err)
@@ -119,10 +101,53 @@ func RefreshWinsService() error {
 
 	defer winSrv.Close()
 
+	// We cannot restart a service which another service depends on.
+	// In the event that we need to update the rancher-wins config file
+	// (and thus restart the rancher-wins service),
+	// we will need to temporarily remove the service dependency from
+	// the rke2 service if it exists. This ensures that rancher-wins can be updated
+	// without potentially impacting node functionality due to a restart of rke2.
+
+	rke2Srv, rke2Exists, err := OpenRke2Service()
+	if err != nil {
+		logrus.Errorf("error opening rke2 service while restarting rancher-wins: %v", err)
+	}
+
+	depRemoved := false
+	if rke2Exists {
+		hasDep, err := rke2Srv.HasRancherWinsServiceDependency()
+		if err != nil {
+			return fmt.Errorf("error encountered while temporarily removing rke2 service dependency: %w", err)
+		}
+		if hasDep {
+			logrus.Info("Temporarily removing rke2 service dependency")
+			depRemoved = true
+			err = rke2Srv.RemoveRancherWinsServiceDependency()
+			rke2Srv.Close()
+			if err != nil {
+				return fmt.Errorf("error encountered while temporarily removing rke2 service dependency: %w", err)
+			}
+		}
+	}
+
 	err = winSrv.Restart()
 	if err != nil {
 		return fmt.Errorf("failed to restart the %s service: %w", winSrv.Name, err)
 	}
 
+	if depRemoved {
+		// if we removed the dependency then we know the service exists
+		rke2Srv, _, err = OpenRke2Service()
+		if err != nil {
+			logrus.Errorf("error opening rke2 service while restarting rancher-wins: %v", err)
+		}
+		logrus.Info("Restoring rke2 service dependency")
+		err = rke2Srv.AddRancherWinsServiceDependency()
+		rke2Srv.Close()
+		if err != nil {
+			return fmt.Errorf("error encountered while restoring rke2 service dependency: %w", err)
+		}
+	}
+
 	return nil
 }

suc/pkg/service/service.go

@@ -3,15 +3,20 @@ package service
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/pkg/errors"
-	"golang.org/x/sys/windows"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/windows"
 	"golang.org/x/sys/windows/svc"
 	"golang.org/x/sys/windows/svc/mgr"
 )
 
+const (
+	stateTransitionAttempts       = 5
+	stateTransitionDelayInSeconds = 5
+)
+
 // Service is a wrapper around a mgr.Service which simplifies
 // common operations and bundles relevant configuration information.
 type Service struct {
@@ -24,7 +29,7 @@ type Service struct {
 // If the provided service does not exist, a nil error and a false boolean will be returned.
 // The caller of Open is responsible for closing the returned Service (via Service.Close()).
 func Open(name string) (service *Service, serviceExists bool, err error) {
-	logrus.Infof("Opening %s service", name)
+	logrus.Debugf("Opening %s service", name)
 	svcMgr, err := mgr.Connect()
 	if err != nil {
 		return nil, false, fmt.Errorf("failed to connect to service manager: %w", err)
@@ -87,7 +92,7 @@ func (s *Service) Restart() error {
 		return fmt.Errorf("failed to start the %s service while attempting to restart: %w", s.Name, err)
 	}
 
-	return s.WaitForState(svc.Running, 5, 5)
+	return s.WaitForState(svc.Running, stateTransitionDelayInSeconds, stateTransitionAttempts)
 }
 
 // Stop sends a svc.Stop control signal to the Service and waits
@@ -98,7 +103,7 @@ func (s *Service) Stop() error {
 		return fmt.Errorf("failed to send Stop signal to %s: %w", s.Name, err)
 	}
 
-	return s.WaitForState(svc.Stopped, 5, 5)
+	return s.WaitForState(svc.Stopped, stateTransitionDelayInSeconds, stateTransitionAttempts)
 }
 
 // Close closes the Service
@@ -113,6 +118,8 @@ func (s *Service) WaitForState(desiredState svc.State, delayInSeconds time.Durat
 	var err error
 	var state svc.State
 
+	logrus.Infof("Waiting for service %s to enter state %s", s.Name, serviceStateToString(desiredState))
+
 	for i := 0; i < maxAttempts; i++ {
 		state, err = s.GetState()
 		if err != nil {
@@ -134,7 +141,9 @@ func (s *Service) WaitForState(desiredState svc.State, delayInSeconds time.Durat
 	return nil
 }
 
-// UpdateConfig commits the stored Service.Config to the registry
+// UpdateConfig commits the stored Service.Config to the registry. Note that
+// the config can only be updated a single time after a service has been opened.
+// In order to update the config again, the service must be closed and reopened.
 func (s *Service) UpdateConfig() error {
 	j, _ := json.MarshalIndent(s.Config, "", " ")
 	logrus.Debugf("Updating config for %s service. Config to be saved:\n%s ", s.Name, string(j))

suc/pkg/service/service_rancher_wins.go

@@ -0,0 +1,45 @@
+package service
+
+import (
+	"fmt"
+
+	"github.com/rancher/wins/pkg/defaults"
+	"github.com/sirupsen/logrus"
+)
+
+type RancherWinsService struct {
+	Service
+}
+
+func OpenRancherWinsService() (*RancherWinsService, bool, error) {
+
+	winsSvc, exists, err := Open(defaults.WindowsServiceName)
+	if err != nil {
+		return nil, false, fmt.Errorf("error encountered opening %s service: %w", defaults.WindowsServiceName, err)
+	}
+
+	if !exists {
+		return nil, false, fmt.Errorf("%s service does not exist", defaults.WindowsServiceName)
+	}
+
+	x := &RancherWinsService{
+		*winsSvc,
+	}
+
+	return x, exists, nil
+}
+
+func (rw *RancherWinsService) ConfigureDelayedStart(enabled bool) error {
+	logrus.Infof("%s service has delayed auto start configured: %t", defaults.WindowsServiceName, rw.Config.DelayedAutoStart)
+	if rw.Config.DelayedAutoStart != enabled {
+		logrus.Infof("updating %s delayed auto start setting to %t", defaults.WindowsServiceName, enabled)
+		rw.Config.DelayedAutoStart = enabled
+		err := rw.UpdateConfig()
+		if err != nil {
+			return fmt.Errorf("failed to update %s service configuration while configuring service start type: %w", defaults.WindowsServiceName, err)
+		}
+	} else {
+		logrus.Infof("%s delayed start already set to %t, nothing to do", defaults.WindowsServiceName, enabled)
+	}
+	return nil
+}