hybrid deploy

Signed-off-by: Lyndon-Li <[email protected]>
vmware-tanzu · Dec 10, 2024 · 2ed820d · 2ed820d
1 parent 0ea4eb5
commit 2ed820d
Show file tree

Hide file tree

Showing 8 changed files with 138 additions and 19 deletions.
diff --git a/changelogs/unreleased/8504-Lyndon-Li b/changelogs/unreleased/8504-Lyndon-Li
@@ -0,0 +1 @@
+Fix issue #8416, #8417, deploy Velero server and node-agent in linux/Windows hybrid env
diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go
@@ -394,7 +394,12 @@ func (o *Options) Run(c *cobra.Command, f client.Factory) error {
 
 		if o.UseNodeAgent {
 			fmt.Println("Waiting for node-agent daemonset to be ready.")
-			if _, err = install.DaemonSetIsReady(dynamicFactory, o.Namespace); err != nil {
+			if _, err = install.NodeAgentIsReady(dynamicFactory, o.Namespace); err != nil {
+				return errors.Wrap(err, errorMsg)
+			}
+
+			fmt.Println("Waiting for node-agent-windows daemonset to be ready.")
+			if _, err = install.NodeAgentWindowsIsReady(dynamicFactory, o.Namespace); err != nil {
 				return errors.Wrap(err, errorMsg)
 			}
 		}

diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go
@@ -200,13 +200,31 @@ func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, confi
 			},
 		},
 	}
-	mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{
-		Scheme: scheme,
-		Cache:  cacheOption,
-	})
+
+	var mgr manager.Manager
+	retry := 10
+	for {
+		mgr, err = ctrl.NewManager(clientConfig, ctrl.Options{
+			Scheme: scheme,
+			Cache:  cacheOption,
+		})
+		if err == nil {
+			break
+		}
+
+		retry--
+		if retry == 0 {
+			break
+		}
+
+		logger.WithError(err).Warn("Failed to create controller manager, need retry")
+
+		time.Sleep(time.Second)
+	}
+
 	if err != nil {
 		cancelFunc()
-		return nil, err
+		return nil, errors.Wrap(err, "error creating controller manager")
 	}
 
 	s := &nodeAgentServer{

diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go
@@ -239,17 +239,34 @@ func newServer(f client.Factory, config *config.Config, logger *logrus.Logger) (
 
 	ctrl.SetLogger(logrusr.New(logger))
 
-	mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{
-		Scheme: scheme,
-		Cache: cache.Options{
-			DefaultNamespaces: map[string]cache.Config{
-				f.Namespace(): {},
+	var mgr manager.Manager
+	retry := 10
+	for {
+		mgr, err = ctrl.NewManager(clientConfig, ctrl.Options{
+			Scheme: scheme,
+			Cache: cache.Options{
+				DefaultNamespaces: map[string]cache.Config{
+					f.Namespace(): {},
+				},
 			},
-		},
-	})
+		})
+		if err == nil {
+			break
+		}
+
+		retry--
+		if retry == 0 {
+			break
+		}
+
+		logger.WithError(err).Warn("Failed to create controller manager, need retry")
+
+		time.Sleep(time.Second)
+	}
+
 	if err != nil {
 		cancelFunc()
-		return nil, err
+		return nil, errors.Wrap(err, "error creating controller manager")
 	}
 
 	credentialFileStore, err := credentials.NewNamespacedFileStore(

diff --git a/pkg/install/daemonset_test.go b/pkg/install/daemonset_test.go
@@ -24,10 +24,23 @@ import (
 )
 
 func TestDaemonSet(t *testing.T) {
+	userID := int64(0)
+	boolFalse := false
+	boolTrue := true
+
 	ds := DaemonSet("velero")
 
 	assert.Equal(t, "node-agent", ds.Spec.Template.Spec.Containers[0].Name)
 	assert.Equal(t, "velero", ds.ObjectMeta.Namespace)
+	assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["name"])
+	assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["role"])
+	assert.Equal(t, "linux", ds.Spec.Template.Spec.NodeSelector["kubernetes.io/os"])
+	assert.Equal(t, "linux", string(ds.Spec.Template.Spec.OS.Name))
+	assert.Equal(t, corev1.PodSecurityContext{RunAsUser: &userID}, *ds.Spec.Template.Spec.SecurityContext)
+	assert.Equal(t, corev1.SecurityContext{Privileged: &boolFalse}, *ds.Spec.Template.Spec.Containers[0].SecurityContext)
+
+	ds = DaemonSet("velero", WithPrivilegedNodeAgent(true))
+	assert.Equal(t, corev1.SecurityContext{Privileged: &boolTrue}, *ds.Spec.Template.Spec.Containers[0].SecurityContext)
 
 	ds = DaemonSet("velero", WithImage("velero/velero:v0.11"))
 	assert.Equal(t, "velero/velero:v0.11", ds.Spec.Template.Spec.Containers[0].Image)
@@ -47,4 +60,14 @@ func TestDaemonSet(t *testing.T) {
 
 	ds = DaemonSet("velero", WithServiceAccountName("test-sa"))
 	assert.Equal(t, "test-sa", ds.Spec.Template.Spec.ServiceAccountName)
+
+	ds = DaemonSet("velero", WithForWinows())
+	assert.Equal(t, "node-agent-windows", ds.Spec.Template.Spec.Containers[0].Name)
+	assert.Equal(t, "velero", ds.ObjectMeta.Namespace)
+	assert.Equal(t, "node-agent-windows", ds.Spec.Template.ObjectMeta.Labels["name"])
+	assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["role"])
+	assert.Equal(t, "windows", ds.Spec.Template.Spec.NodeSelector["kubernetes.io/os"])
+	assert.Equal(t, "windows", string(ds.Spec.Template.Spec.OS.Name))
+	assert.Equal(t, (*corev1.PodSecurityContext)(nil), ds.Spec.Template.Spec.SecurityContext)
+	assert.Equal(t, (*corev1.SecurityContext)(nil), ds.Spec.Template.Spec.Containers[0].SecurityContext)
 }
diff --git a/pkg/install/install.go b/pkg/install/install.go
@@ -206,9 +206,19 @@ func DeploymentIsReady(factory client.DynamicFactory, namespace string) (bool, e
 	return isReady, err
 }
 
-// DaemonSetIsReady will poll the Kubernetes API server to ensure the node-agent daemonset is ready, i.e. that
+// NodeAgentIsReady will poll the Kubernetes API server to ensure the node-agent daemonset is ready, i.e. that
 // pods are scheduled and available on all of the desired nodes.
-func DaemonSetIsReady(factory client.DynamicFactory, namespace string) (bool, error) {
+func NodeAgentIsReady(factory client.DynamicFactory, namespace string) (bool, error) {
+	return daemonSetIsReady(factory, namespace, "node-agent")
+}
+
+// NodeAgentWindowsIsReady will poll the Kubernetes API server to ensure the node-agent-windows daemonset is ready, i.e. that
+// pods are scheduled and available on all of the desired nodes.
+func NodeAgentWindowsIsReady(factory client.DynamicFactory, namespace string) (bool, error) {
+	return daemonSetIsReady(factory, namespace, "node-agent-windows")
+}
+
+func daemonSetIsReady(factory client.DynamicFactory, namespace string, name string) (bool, error) {
 	gvk := schema.FromAPIVersionAndKind(appsv1.SchemeGroupVersion.String(), "DaemonSet")
 	apiResource := metav1.APIResource{
 		Name:       "daemonsets",
@@ -225,7 +235,7 @@ func DaemonSetIsReady(factory client.DynamicFactory, namespace string) (bool, er
 	var readyObservations int32
 
 	err = wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) {
-		unstructuredDaemonSet, err := c.Get("node-agent", metav1.GetOptions{})
+		unstructuredDaemonSet, err := c.Get(name, metav1.GetOptions{})
 		if apierrors.IsNotFound(err) {
 			return false, nil
 		} else if err != nil {

diff --git a/pkg/install/install_test.go b/pkg/install/install_test.go
@@ -127,7 +127,7 @@ func TestDeploymentIsReady(t *testing.T) {
 	assert.True(t, ready)
 }
 
-func TestDaemonSetIsReady(t *testing.T) {
+func TestNodeAgentIsReady(t *testing.T) {
 	daemonset := &appsv1.DaemonSet{
 		Status: appsv1.DaemonSetStatus{
 			NumberAvailable:        1,
@@ -143,7 +143,28 @@ func TestDaemonSetIsReady(t *testing.T) {
 	factory := &test.FakeDynamicFactory{}
 	factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil)
 
-	ready, err := DaemonSetIsReady(factory, "velero")
+	ready, err := NodeAgentIsReady(factory, "velero")
+	require.NoError(t, err)
+	assert.True(t, ready)
+}
+
+func TestNodeAgentWindowsIsReady(t *testing.T) {
+	daemonset := &appsv1.DaemonSet{
+		Status: appsv1.DaemonSetStatus{
+			NumberAvailable:        0,
+			DesiredNumberScheduled: 0,
+		},
+	}
+	obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(daemonset)
+	require.NoError(t, err)
+
+	dc := &test.FakeDynamicClient{}
+	dc.On("Get", mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: obj}, nil)
+
+	factory := &test.FakeDynamicFactory{}
+	factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil)
+
+	ready, err := NodeAgentWindowsIsReady(factory, "velero")
 	require.NoError(t, err)
 	assert.True(t, ready)
 }
diff --git a/pkg/plugin/clientmgmt/process/registry_test.go b/pkg/plugin/clientmgmt/process/registry_test.go
@@ -45,16 +45,22 @@ func TestNewRegistry(t *testing.T) {
 
 type fakeFileInfo struct {
 	fs.FileInfo
+	name string
 	mode os.FileMode
 }
 
 func (f *fakeFileInfo) Mode() os.FileMode {
 	return f.mode
 }
 
+func (f *fakeFileInfo) Name() string {
+	return f.name
+}
+
 func TestExecutable(t *testing.T) {
 	tests := []struct {
 		name             string
+		fileName         string
 		mode             uint32
 		expectExecutable bool
 	}{
@@ -90,11 +96,29 @@ func TestExecutable(t *testing.T) {
 			mode:             0777,
 			expectExecutable: true,
 		},
+		{
+			name:             "windows lower case",
+			fileName:         "test.exe",
+			mode:             0,
+			expectExecutable: true,
+		},
+		{
+			name:             "windows upper case",
+			fileName:         "test.EXE",
+			mode:             0,
+			expectExecutable: true,
+		},
+		{
+			name:     "windows wrong ext",
+			fileName: "test.EXE1",
+			mode:     0,
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			info := &fakeFileInfo{
+				name: test.fileName,
 				mode: os.FileMode(test.mode),
 			}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Fix issue #8416, #8417, deploy Velero server and node-agent in linux/Windows hybrid env