diff --git a/changelogs/unreleased/8504-Lyndon-Li b/changelogs/unreleased/8504-Lyndon-Li new file mode 100644 index 0000000000..f6e877d954 --- /dev/null +++ b/changelogs/unreleased/8504-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #8416, #8417, deploy Velero server and node-agent in linux/Windows hybrid env \ No newline at end of file diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 45be13d9fc..a72ac59724 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -394,7 +394,12 @@ func (o *Options) Run(c *cobra.Command, f client.Factory) error { if o.UseNodeAgent { fmt.Println("Waiting for node-agent daemonset to be ready.") - if _, err = install.DaemonSetIsReady(dynamicFactory, o.Namespace); err != nil { + if _, err = install.NodeAgentIsReady(dynamicFactory, o.Namespace); err != nil { + return errors.Wrap(err, errorMsg) + } + + fmt.Println("Waiting for node-agent-windows daemonset to be ready.") + if _, err = install.NodeAgentWindowsIsReady(dynamicFactory, o.Namespace); err != nil { return errors.Wrap(err, errorMsg) } } diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go index ba7c1610fb..713d863593 100644 --- a/pkg/cmd/cli/nodeagent/server.go +++ b/pkg/cmd/cli/nodeagent/server.go @@ -200,13 +200,31 @@ func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, confi }, }, } - mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{ - Scheme: scheme, - Cache: cacheOption, - }) + + var mgr manager.Manager + retry := 10 + for { + mgr, err = ctrl.NewManager(clientConfig, ctrl.Options{ + Scheme: scheme, + Cache: cacheOption, + }) + if err == nil { + break + } + + retry-- + if retry == 0 { + break + } + + logger.WithError(err).Warn("Failed to create controller manager, need retry") + + time.Sleep(time.Second) + } + if err != nil { cancelFunc() - return nil, err + return nil, errors.Wrap(err, "error creating controller manager") } s := &nodeAgentServer{ diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index b5f9576f88..a0828e69ef 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -239,17 +239,34 @@ func newServer(f client.Factory, config *config.Config, logger *logrus.Logger) ( ctrl.SetLogger(logrusr.New(logger)) - mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{ - Scheme: scheme, - Cache: cache.Options{ - DefaultNamespaces: map[string]cache.Config{ - f.Namespace(): {}, + var mgr manager.Manager + retry := 10 + for { + mgr, err = ctrl.NewManager(clientConfig, ctrl.Options{ + Scheme: scheme, + Cache: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + f.Namespace(): {}, + }, }, - }, - }) + }) + if err == nil { + break + } + + retry-- + if retry == 0 { + break + } + + logger.WithError(err).Warn("Failed to create controller manager, need retry") + + time.Sleep(time.Second) + } + if err != nil { cancelFunc() - return nil, err + return nil, errors.Wrap(err, "error creating controller manager") } credentialFileStore, err := credentials.NewNamespacedFileStore( diff --git a/pkg/install/daemonset_test.go b/pkg/install/daemonset_test.go index c181306e2c..7eff0ec71f 100644 --- a/pkg/install/daemonset_test.go +++ b/pkg/install/daemonset_test.go @@ -24,10 +24,23 @@ import ( ) func TestDaemonSet(t *testing.T) { + userID := int64(0) + boolFalse := false + boolTrue := true + ds := DaemonSet("velero") assert.Equal(t, "node-agent", ds.Spec.Template.Spec.Containers[0].Name) assert.Equal(t, "velero", ds.ObjectMeta.Namespace) + assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["name"]) + assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["role"]) + assert.Equal(t, "linux", ds.Spec.Template.Spec.NodeSelector["kubernetes.io/os"]) + assert.Equal(t, "linux", string(ds.Spec.Template.Spec.OS.Name)) + assert.Equal(t, corev1.PodSecurityContext{RunAsUser: &userID}, *ds.Spec.Template.Spec.SecurityContext) + assert.Equal(t, corev1.SecurityContext{Privileged: &boolFalse}, *ds.Spec.Template.Spec.Containers[0].SecurityContext) + + ds = DaemonSet("velero", WithPrivilegedNodeAgent(true)) + assert.Equal(t, corev1.SecurityContext{Privileged: &boolTrue}, *ds.Spec.Template.Spec.Containers[0].SecurityContext) ds = DaemonSet("velero", WithImage("velero/velero:v0.11")) assert.Equal(t, "velero/velero:v0.11", ds.Spec.Template.Spec.Containers[0].Image) @@ -47,4 +60,14 @@ func TestDaemonSet(t *testing.T) { ds = DaemonSet("velero", WithServiceAccountName("test-sa")) assert.Equal(t, "test-sa", ds.Spec.Template.Spec.ServiceAccountName) + + ds = DaemonSet("velero", WithForWinows()) + assert.Equal(t, "node-agent-windows", ds.Spec.Template.Spec.Containers[0].Name) + assert.Equal(t, "velero", ds.ObjectMeta.Namespace) + assert.Equal(t, "node-agent-windows", ds.Spec.Template.ObjectMeta.Labels["name"]) + assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["role"]) + assert.Equal(t, "windows", ds.Spec.Template.Spec.NodeSelector["kubernetes.io/os"]) + assert.Equal(t, "windows", string(ds.Spec.Template.Spec.OS.Name)) + assert.Equal(t, (*corev1.PodSecurityContext)(nil), ds.Spec.Template.Spec.SecurityContext) + assert.Equal(t, (*corev1.SecurityContext)(nil), ds.Spec.Template.Spec.Containers[0].SecurityContext) } diff --git a/pkg/install/install.go b/pkg/install/install.go index 3bf0651702..391b97ec1b 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -206,9 +206,19 @@ func DeploymentIsReady(factory client.DynamicFactory, namespace string) (bool, e return isReady, err } -// DaemonSetIsReady will poll the Kubernetes API server to ensure the node-agent daemonset is ready, i.e. that +// NodeAgentIsReady will poll the Kubernetes API server to ensure the node-agent daemonset is ready, i.e. that // pods are scheduled and available on all of the desired nodes. -func DaemonSetIsReady(factory client.DynamicFactory, namespace string) (bool, error) { +func NodeAgentIsReady(factory client.DynamicFactory, namespace string) (bool, error) { + return daemonSetIsReady(factory, namespace, "node-agent") +} + +// NodeAgentWindowsIsReady will poll the Kubernetes API server to ensure the node-agent-windows daemonset is ready, i.e. that +// pods are scheduled and available on all of the desired nodes. +func NodeAgentWindowsIsReady(factory client.DynamicFactory, namespace string) (bool, error) { + return daemonSetIsReady(factory, namespace, "node-agent-windows") +} + +func daemonSetIsReady(factory client.DynamicFactory, namespace string, name string) (bool, error) { gvk := schema.FromAPIVersionAndKind(appsv1.SchemeGroupVersion.String(), "DaemonSet") apiResource := metav1.APIResource{ Name: "daemonsets", @@ -225,7 +235,7 @@ func DaemonSetIsReady(factory client.DynamicFactory, namespace string) (bool, er var readyObservations int32 err = wait.PollUntilContextTimeout(context.Background(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { - unstructuredDaemonSet, err := c.Get("node-agent", metav1.GetOptions{}) + unstructuredDaemonSet, err := c.Get(name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { return false, nil } else if err != nil { diff --git a/pkg/install/install_test.go b/pkg/install/install_test.go index 52bfbaf418..e9c39adf18 100644 --- a/pkg/install/install_test.go +++ b/pkg/install/install_test.go @@ -127,7 +127,7 @@ func TestDeploymentIsReady(t *testing.T) { assert.True(t, ready) } -func TestDaemonSetIsReady(t *testing.T) { +func TestNodeAgentIsReady(t *testing.T) { daemonset := &appsv1.DaemonSet{ Status: appsv1.DaemonSetStatus{ NumberAvailable: 1, @@ -143,7 +143,28 @@ func TestDaemonSetIsReady(t *testing.T) { factory := &test.FakeDynamicFactory{} factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) - ready, err := DaemonSetIsReady(factory, "velero") + ready, err := NodeAgentIsReady(factory, "velero") + require.NoError(t, err) + assert.True(t, ready) +} + +func TestNodeAgentWindowsIsReady(t *testing.T) { + daemonset := &appsv1.DaemonSet{ + Status: appsv1.DaemonSetStatus{ + NumberAvailable: 0, + DesiredNumberScheduled: 0, + }, + } + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(daemonset) + require.NoError(t, err) + + dc := &test.FakeDynamicClient{} + dc.On("Get", mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: obj}, nil) + + factory := &test.FakeDynamicFactory{} + factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) + + ready, err := NodeAgentWindowsIsReady(factory, "velero") require.NoError(t, err) assert.True(t, ready) } diff --git a/pkg/plugin/clientmgmt/process/registry_test.go b/pkg/plugin/clientmgmt/process/registry_test.go index 0bcef7c0b4..e188419b2f 100644 --- a/pkg/plugin/clientmgmt/process/registry_test.go +++ b/pkg/plugin/clientmgmt/process/registry_test.go @@ -45,6 +45,7 @@ func TestNewRegistry(t *testing.T) { type fakeFileInfo struct { fs.FileInfo + name string mode os.FileMode } @@ -52,9 +53,14 @@ func (f *fakeFileInfo) Mode() os.FileMode { return f.mode } +func (f *fakeFileInfo) Name() string { + return f.name +} + func TestExecutable(t *testing.T) { tests := []struct { name string + fileName string mode uint32 expectExecutable bool }{ @@ -90,11 +96,29 @@ func TestExecutable(t *testing.T) { mode: 0777, expectExecutable: true, }, + { + name: "windows lower case", + fileName: "test.exe", + mode: 0, + expectExecutable: true, + }, + { + name: "windows upper case", + fileName: "test.EXE", + mode: 0, + expectExecutable: true, + }, + { + name: "windows wrong ext", + fileName: "test.EXE1", + mode: 0, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { info := &fakeFileInfo{ + name: test.fileName, mode: os.FileMode(test.mode), }