From d53b3821d7f517c428a737199ca388f4a19d7d1d Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Tue, 19 Nov 2024 15:07:06 +0800 Subject: [PATCH] Modify the schedule cases. * Prettify the code. * Replace math/rand with crypto/rand * Replace PollUnitl with PollUntilContextTimeout Signed-off-by: Xun Jiang --- test/e2e/e2e_suite_test.go | 2 +- test/e2e/resource-filtering/exclude_label.go | 2 +- test/e2e/schedule/ordered_resources.go | 187 +++++++++++------ test/e2e/schedule/schedule-backup-creation.go | 87 ++++++-- test/e2e/schedule/schedule.go | 196 ++++++++++++++---- test/util/k8s/configmap.go | 2 +- test/util/velero/install.go | 2 +- test/util/velero/velero_utils.go | 18 +- 8 files changed, 359 insertions(+), 137 deletions(-) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 790019ba293..d7339ad065e 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -548,7 +548,7 @@ var _ = Describe( ) var _ = Describe( "Backup resources should follow the specific order in schedule", - Label("Schedule", "OrderedResources", "LongTime"), + Label("Schedule", "OrderedResources"), ScheduleOrderedResources, ) var _ = Describe( diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index 0dd1753694a..b90d63dd878 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -144,7 +144,7 @@ func (e *ExcludeFromBackup) Verify() error { Expect(apierrors.IsNotFound(err)).To(BeTrue()) //Check configmap: should be included - _, err = GetConfigmap(e.Client.ClientGo, namespace, e.CaseBaseName) + _, err = GetConfigMap(e.Client.ClientGo, namespace, e.CaseBaseName) Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to list configmap in namespace: %q", namespace)) }) return nil diff --git a/test/e2e/schedule/ordered_resources.go b/test/e2e/schedule/ordered_resources.go index 6df2ab17d1b..d5b3fa33fa8 100644 --- a/test/e2e/schedule/ordered_resources.go +++ b/test/e2e/schedule/ordered_resources.go @@ -18,6 +18,7 @@ limitations under the License. //the ordered resources test related to https://github.com/vmware-tanzu/velero/issues/4561 import ( + "context" "fmt" "strings" "time" @@ -29,85 +30,112 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - . "github.com/vmware-tanzu/velero/test/e2e/test" - . "github.com/vmware-tanzu/velero/test/util/k8s" - . "github.com/vmware-tanzu/velero/test/util/velero" + framework "github.com/vmware-tanzu/velero/test/e2e/test" + k8sutil "github.com/vmware-tanzu/velero/test/util/k8s" + veleroutil "github.com/vmware-tanzu/velero/test/util/velero" ) -var ScheduleOrderedResources func() = TestFunc(&OrderedResources{}) +var ScheduleOrderedResources func() = framework.TestFunc(&OrderedResources{}) type OrderedResources struct { - Namespace string - ScheduleName string - OrderMap map[string]string - ScheduleArgs []string - TestCase + Namespace string + ScheduleName string + OrderResource map[string]string + ScheduleArgs []string + framework.TestCase } func (o *OrderedResources) Init() error { - o.TestCase.Init() + Expect(o.TestCase.Init()).To(Succeed()) + o.CaseBaseName = "ordered-resources-" + o.UUIDgen o.ScheduleName = "schedule-" + o.CaseBaseName o.Namespace = o.CaseBaseName + "-" + o.UUIDgen - o.OrderMap = map[string]string{ + + o.OrderResource = map[string]string{ "deployments": fmt.Sprintf("deploy-%s", o.CaseBaseName), "secrets": fmt.Sprintf("secret-%s", o.CaseBaseName), "configmaps": fmt.Sprintf("configmap-%s", o.CaseBaseName), } - o.TestMsg = &TestMSG{ + + orderResourceArray := make([]string, 0) + for k, v := range o.OrderResource { + orderResourceArray = append( + orderResourceArray, + fmt.Sprintf("%s=%s", k, v), + ) + } + orderResourceStr := strings.Join(orderResourceArray, ";") + + o.TestMsg = &framework.TestMSG{ Desc: "Create a schedule to backup resources in a specific order should be successful", FailedMSG: "Failed to verify schedule backup resources in a specific order", Text: "Create a schedule to backup resources in a specific order should be successful", } - o.ScheduleArgs = []string{"--schedule", "@every 1m", - "--include-namespaces", o.Namespace, "--default-volumes-to-fs-backup", "--ordered-resources"} - var orderStr string - for kind, resource := range o.OrderMap { - orderStr += fmt.Sprintf("%s=%s;", kind, resource) + + o.ScheduleArgs = []string{ + "--schedule", + "@every 1m", + "--include-namespaces", + o.Namespace, + "--default-volumes-to-fs-backup", + "--ordered-resources", + orderResourceStr, } - o.ScheduleArgs = append(o.ScheduleArgs, strings.TrimRight(orderStr, ";")) return nil } + func (o *OrderedResources) CreateResources() error { label := map[string]string{ "orderedresources": "true", } fmt.Printf("Creating resources in %s namespace ...\n", o.Namespace) - if err := CreateNamespace(o.Ctx, o.Client, o.Namespace); err != nil { + if err := k8sutil.CreateNamespace(o.Ctx, o.Client, o.Namespace); err != nil { return errors.Wrapf(err, "failed to create namespace %s", o.Namespace) } + //Create deployment deploymentName := fmt.Sprintf("deploy-%s", o.CaseBaseName) fmt.Printf("Creating deployment %s in %s namespaces ...\n", deploymentName, o.Namespace) - deployment := NewDeployment(deploymentName, o.Namespace, 1, label, nil).Result() - deployment, err := CreateDeployment(o.Client.ClientGo, o.Namespace, deployment) + deployment := k8sutil.NewDeployment(deploymentName, o.Namespace, 1, label, nil).Result() + _, err := k8sutil.CreateDeployment(o.Client.ClientGo, o.Namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to create namespace %q with err %v", o.Namespace, err)) } - err = WaitForReadyDeployment(o.Client.ClientGo, o.Namespace, deployment.Name) + err = k8sutil.WaitForReadyDeployment(o.Client.ClientGo, o.Namespace, deployment.Name) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", o.Namespace)) } + //Create Secret secretName := fmt.Sprintf("secret-%s", o.CaseBaseName) fmt.Printf("Creating secret %s in %s namespaces ...\n", secretName, o.Namespace) - _, err = CreateSecret(o.Client.ClientGo, o.Namespace, secretName, label) + _, err = k8sutil.CreateSecret(o.Client.ClientGo, o.Namespace, secretName, label) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to create secret in the namespace %q", o.Namespace)) } - err = WaitForSecretsComplete(o.Client.ClientGo, o.Namespace, secretName) + err = k8sutil.WaitForSecretsComplete(o.Client.ClientGo, o.Namespace, secretName) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", o.Namespace)) } - //Create Configmap - configmapName := fmt.Sprintf("configmap-%s", o.CaseBaseName) - fmt.Printf("Creating configmap %s in %s namespaces ...\n", configmapName, o.Namespace) - _, err = CreateConfigMap(o.Client.ClientGo, o.Namespace, configmapName, label, nil) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to create configmap in the namespace %q", o.Namespace)) + + //Create ConfigMap + cmName := fmt.Sprintf("configmap-%s", o.CaseBaseName) + fmt.Printf("Creating ConfigMap %s in %s namespaces ...\n", cmName, o.Namespace) + if _, err := k8sutil.CreateConfigMap( + o.Client.ClientGo, + o.Namespace, + cmName, + label, + nil, + ); err != nil { + return errors.Wrap( + err, + fmt.Sprintf("failed to create ConfigMap in the namespace %q", o.Namespace), + ) } - err = WaitForConfigMapComplete(o.Client.ClientGo, o.Namespace, configmapName) + err = k8sutil.WaitForConfigMapComplete(o.Client.ClientGo, o.Namespace, cmName) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", o.Namespace)) } @@ -116,39 +144,75 @@ func (o *OrderedResources) CreateResources() error { func (o *OrderedResources) Backup() error { By(fmt.Sprintf("Create schedule the workload in %s namespace", o.Namespace), func() { - err := VeleroScheduleCreate(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, o.ScheduleName, o.ScheduleArgs) + err := veleroutil.VeleroScheduleCreate( + o.Ctx, + o.VeleroCfg.VeleroCLI, + o.VeleroCfg.VeleroNamespace, + o.ScheduleName, + o.ScheduleArgs, + ) Expect(err).To(Succeed(), fmt.Sprintf("Failed to create schedule %s with err %v", o.ScheduleName, err)) }) - return nil -} -func (o *OrderedResources) Destroy() error { + By("Sleep 1 minute to wait the schedule triggers the backup.", func() { + time.Sleep(1 * time.Minute) + }) return nil } func (o *OrderedResources) Verify() error { - By(fmt.Sprintf("Checking resource order in %s schedule cr", o.ScheduleName), func() { - err := CheckScheduleWithResourceOrder(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, o.ScheduleName, o.OrderMap) - Expect(err).To(Succeed(), fmt.Sprintf("Failed to check schedule %s with err %v", o.ScheduleName, err)) + By(fmt.Sprintf("Checking resource order in %s schedule CR", o.ScheduleName), func() { + err := veleroutil.CheckScheduleWithResourceOrder( + o.Ctx, + o.VeleroCfg.VeleroCLI, + o.VeleroCfg.VeleroNamespace, + o.ScheduleName, + o.OrderResource, + ) + Expect(err).To( + Succeed(), + fmt.Sprintf("Failed to check schedule %s with err %v", o.ScheduleName, err), + ) }) By("Checking resource order in backup cr", func() { backupList := new(velerov1api.BackupList) - err := waitutil.PollImmediate(10*time.Second, time.Minute*5, func() (bool, error) { - if err := o.Client.Kubebuilder.List(o.Ctx, backupList, &kbclient.ListOptions{Namespace: o.VeleroCfg.VeleroNamespace}); err != nil { - return false, fmt.Errorf("failed to list backup object in %s namespace with err %v", o.VeleroCfg.VeleroNamespace, err) - } - - for _, backup := range backupList.Items { - if err := CheckBackupWithResourceOrder(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, backup.Name, o.OrderMap); err == nil { - return true, nil + err := waitutil.PollUntilContextTimeout( + o.Ctx, + 10*time.Second, + time.Minute*5, + true, + func(ctx context.Context) (bool, error) { + if err := o.Client.Kubebuilder.List( + o.Ctx, + backupList, + &kbclient.ListOptions{Namespace: o.VeleroCfg.VeleroNamespace}, + ); err != nil { + return false, fmt.Errorf("failed to list backup object in %s namespace with err %v", + o.VeleroCfg.VeleroNamespace, err) + } + + for _, backup := range backupList.Items { + if err := veleroutil.CheckBackupWithResourceOrder( + o.Ctx, + o.VeleroCfg.VeleroCLI, + o.VeleroCfg.VeleroNamespace, + backup.Name, + o.OrderResource, + ); err == nil { + return true, nil + } } - } - fmt.Printf("still finding backup created by schedule %s ...\n", o.ScheduleName) - return false, nil - }) - Expect(err).To(Succeed(), fmt.Sprintf("Failed to check schedule %s created backup with err %v", o.ScheduleName, err)) + fmt.Printf("still finding backup created by schedule %s ...\n", o.ScheduleName) + return false, nil + }) + Expect(err).To( + Succeed(), + fmt.Sprintf("Failed to check schedule %s created backup with err %v", + o.ScheduleName, err), + ) }) + return nil } @@ -156,22 +220,15 @@ func (o *OrderedResources) Clean() error { if CurrentSpecReport().Failed() && o.VeleroCfg.FailFast { fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } else { - Expect(VeleroScheduleDelete(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, o.ScheduleName)).To(Succeed()) + Expect(veleroutil.VeleroScheduleDelete( + o.Ctx, + o.VeleroCfg.VeleroCLI, + o.VeleroCfg.VeleroNamespace, + o.ScheduleName, + )).To(Succeed()) + Expect(o.TestCase.Clean()).To(Succeed()) } return nil } - -func (o *OrderedResources) DeleteAllBackups() error { - backupList := new(velerov1api.BackupList) - if err := o.Client.Kubebuilder.List(o.Ctx, backupList, &kbclient.ListOptions{Namespace: o.VeleroCfg.VeleroNamespace}); err != nil { - return fmt.Errorf("failed to list backup object in %s namespace with err %v", o.VeleroCfg.VeleroNamespace, err) - } - for _, backup := range backupList.Items { - if err := VeleroBackupDelete(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, backup.Name); err != nil { - return err - } - } - return nil -} diff --git a/test/e2e/schedule/schedule-backup-creation.go b/test/e2e/schedule/schedule-backup-creation.go index df30b1752f6..2f5c8d22047 100644 --- a/test/e2e/schedule/schedule-backup-creation.go +++ b/test/e2e/schedule/schedule-backup-creation.go @@ -2,8 +2,9 @@ package schedule import ( "context" + "crypto/rand" "fmt" - "math/rand" + "math/big" "strings" "time" @@ -21,7 +22,7 @@ type ScheduleBackupCreation struct { namespace string ScheduleName string ScheduleArgs []string - Period int //Limitation: The unit is minitue only and 60 is divisible by it + Period int // The minimum unit is minute. randBackupName string verifyTimes int volume string @@ -34,7 +35,8 @@ type ScheduleBackupCreation struct { var ScheduleBackupCreationTest func() = framework.TestFunc(&ScheduleBackupCreation{}) func (s *ScheduleBackupCreation) Init() error { - s.TestCase.Init() + Expect(s.TestCase.Init()).To(Succeed()) + s.CaseBaseName = "schedule-backup-creation-test" + s.UUIDgen s.ScheduleName = "schedule-" + s.CaseBaseName s.namespace = s.GetTestCase().CaseBaseName @@ -65,14 +67,35 @@ func (s *ScheduleBackupCreation) Init() error { func (s *ScheduleBackupCreation) CreateResources() error { By(fmt.Sprintf("Create namespace %s", s.namespace), func() { - Expect(k8sutil.CreateNamespace(s.Ctx, s.Client, s.namespace)).To(Succeed(), + Expect( + k8sutil.CreateNamespace( + s.Ctx, + s.Client, + s.namespace, + ), + ).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", s.namespace)) }) By(fmt.Sprintf("Create pod %s in namespace %s", s.podName, s.namespace), func() { - _, err := k8sutil.CreatePod(s.Client, s.namespace, s.podName, test.StorageClassName, s.pvcName, []string{s.volume}, nil, s.podAnn) + _, err := k8sutil.CreatePod( + s.Client, + s.namespace, + s.podName, + test.StorageClassName, + s.pvcName, + []string{s.volume}, + nil, + s.podAnn, + ) Expect(err).To(Succeed()) - err = k8sutil.WaitForPods(s.Ctx, s.Client, s.namespace, []string{s.podName}) + + err = k8sutil.WaitForPods( + s.Ctx, + s.Client, + s.namespace, + []string{s.podName}, + ) Expect(err).To(Succeed()) }) return nil @@ -80,18 +103,35 @@ func (s *ScheduleBackupCreation) CreateResources() error { func (s *ScheduleBackupCreation) Backup() error { // Wait until the beginning of the given period to create schedule, it will give us - // a predictable period to wait for the first scheduled backup, and verify no immediate - // scheduled backup was created between schedule creation and first scheduled backup. + // a predictable period to wait for the first scheduled backup, and verify no immediate + // scheduled backup was created between schedule creation and first scheduled backup. By(fmt.Sprintf("Creating schedule %s ......\n", s.ScheduleName), func() { for i := 0; i < s.Period*60/30; i++ { time.Sleep(30 * time.Second) now := time.Now().Minute() triggerNow := now % s.Period if triggerNow == 0 { - Expect(veleroutil.VeleroScheduleCreate(s.Ctx, s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, s.ScheduleName, s.ScheduleArgs)).To(Succeed(), func() string { - veleroutil.RunDebug(context.Background(), s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, "", "") - return "Fail to create schedule" - }) + Expect( + veleroutil.VeleroScheduleCreate( + s.Ctx, + s.VeleroCfg.VeleroCLI, + s.VeleroCfg.VeleroNamespace, + s.ScheduleName, + s.ScheduleArgs, + ), + ).To( + Succeed(), + func() string { + veleroutil.RunDebug( + context.Background(), + s.VeleroCfg.VeleroCLI, + s.VeleroCfg.VeleroNamespace, + "", + "", + ) + + return "Fail to create schedule" + }) break } } @@ -103,11 +143,17 @@ func (s *ScheduleBackupCreation) Backup() error { By(fmt.Sprintf("Get backups every %d minute, and backups count should increase 1 more step in the same pace\n", s.Period), func() { for i := 1; i <= s.verifyTimes; i++ { - fmt.Printf("Start to sleep %d minute #%d time...\n", s.podSleepDuration, i) + fmt.Printf("Start to sleep %f minute #%d time...\n", s.podSleepDuration.Minutes(), i) mi, _ := time.ParseDuration("60s") time.Sleep(s.podSleepDuration + mi) bMap := make(map[string]string) - backupsInfo, err := veleroutil.GetScheduledBackupsCreationTime(s.Ctx, s.VeleroCfg.VeleroCLI, "default", s.ScheduleName) + + backupsInfo, err := veleroutil.GetScheduledBackupsCreationTime( + s.Ctx, + s.VeleroCfg.VeleroCLI, + "default", + s.ScheduleName, + ) Expect(err).To(Succeed()) Expect(backupsInfo).To(HaveLen(i)) for index, bi := range backupsInfo { @@ -118,7 +164,9 @@ func (s *ScheduleBackupCreation) Backup() error { Expect(err).To(Succeed()) } if i == s.verifyTimes-1 { - backupInfo := backupsInfo[rand.Intn(len(backupsInfo))] + randomNum, err := rand.Int(rand.Reader, big.NewInt(int64(len(backupsInfo)))) + Expect(err).To(Succeed()) + backupInfo := backupsInfo[randomNum.Int64()] s.randBackupName = strings.Split(backupInfo, ",")[0] } } @@ -130,7 +178,14 @@ func (s *ScheduleBackupCreation) Clean() error { if CurrentSpecReport().Failed() && s.VeleroCfg.FailFast { fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } else { - Expect(veleroutil.VeleroScheduleDelete(s.Ctx, s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, s.ScheduleName)).To(Succeed()) + Expect( + veleroutil.VeleroScheduleDelete( + s.Ctx, + s.VeleroCfg.VeleroCLI, + s.VeleroCfg.VeleroNamespace, + s.ScheduleName, + ), + ).To(Succeed()) Expect(s.TestCase.Clean()).To(Succeed()) } diff --git a/test/e2e/schedule/schedule.go b/test/e2e/schedule/schedule.go index f1a4bfe2136..231a69845f0 100644 --- a/test/e2e/schedule/schedule.go +++ b/test/e2e/schedule/schedule.go @@ -2,42 +2,44 @@ package schedule import ( "context" + "crypto/rand" "fmt" - "math/rand" + "math/big" "strings" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "github.com/vmware-tanzu/velero/test/e2e/test" - . "github.com/vmware-tanzu/velero/test/util/k8s" - . "github.com/vmware-tanzu/velero/test/util/velero" + framework "github.com/vmware-tanzu/velero/test/e2e/test" + k8sutil "github.com/vmware-tanzu/velero/test/util/k8s" + veleroutil "github.com/vmware-tanzu/velero/test/util/velero" ) type ScheduleBackup struct { - TestCase + framework.TestCase ScheduleName string ScheduleArgs []string - Period int //Limitation: The unit is minitue only and 60 is divisible by it + Period int // The minimum unit is minute. randBackupName string verifyTimes int } -var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{}) +var ScheduleBackupTest func() = framework.TestFunc(&ScheduleBackup{}) func (n *ScheduleBackup) Init() error { - n.TestCase.Init() + Expect(n.TestCase.Init()).To(Succeed()) + n.CaseBaseName = "schedule-backup-" + n.UUIDgen n.NSIncluded = &[]string{n.CaseBaseName} n.ScheduleName = "schedule-" + n.CaseBaseName n.RestoreName = "restore-" + n.CaseBaseName n.Period = 3 // Unit is minute n.verifyTimes = 5 // More larger verify times more confidence we have - n.TestMsg = &TestMSG{ + n.TestMsg = &framework.TestMSG{ Desc: "Set up a scheduled backup defined by a Cron expression", FailedMSG: "Failed to schedule a backup", - Text: "should backup periodly according to the schedule", + Text: "should backup periodically according to the schedule", } n.ScheduleArgs = []string{ "--include-namespaces", strings.Join(*n.NSIncluded, ","), @@ -47,16 +49,33 @@ func (n *ScheduleBackup) Init() error { Expect(n.Period).To(BeNumerically("<", 30)) return nil } + func (n *ScheduleBackup) CreateResources() error { for _, ns := range *n.NSIncluded { By(fmt.Sprintf("Creating namespaces %s ......\n", ns), func() { - Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) + Expect( + k8sutil.CreateNamespace( + n.Ctx, + n.Client, + ns, + ), + ).To( + Succeed(), + fmt.Sprintf("Failed to create namespace %s", ns), + ) }) - configmaptName := n.CaseBaseName - fmt.Printf("Creating configmap %s in namespaces ...%s\n", configmaptName, ns) - _, err := CreateConfigMap(n.Client.ClientGo, ns, configmaptName, nil, nil) - Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", ns)) - Expect(WaitForConfigMapComplete(n.Client.ClientGo, ns, configmaptName)).To(Succeed(), + + cmName := n.CaseBaseName + fmt.Printf("Creating ConfigMap %s in namespaces ...%s\n", cmName, ns) + _, err := k8sutil.CreateConfigMap( + n.Client.ClientGo, + ns, + cmName, + nil, + nil, + ) + Expect(err).To(Succeed(), fmt.Sprintf("failed to create ConfigMap in the namespace %q", ns)) + Expect(k8sutil.WaitForConfigMapComplete(n.Client.ClientGo, ns, cmName)).To(Succeed(), fmt.Sprintf("failed to ensure secret completion in namespace: %q", ns)) } return nil @@ -64,25 +83,42 @@ func (n *ScheduleBackup) CreateResources() error { func (n *ScheduleBackup) Backup() error { // Wait until the beginning of the given period to create schedule, it will give us - // a predictable period to wait for the first scheduled backup, and verify no immediate - // scheduled backup was created between schedule creation and first scheduled backup. + // a predictable period to wait for the first scheduled backup, and verify no immediate + // scheduled backup was created between schedule creation and first scheduled backup. By(fmt.Sprintf("Creating schedule %s ......\n", n.ScheduleName), func() { for i := 0; i < n.Period*60/30; i++ { time.Sleep(30 * time.Second) + now := time.Now().Minute() triggerNow := now % n.Period if triggerNow == 0 { - Expect(VeleroScheduleCreate(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName, n.ScheduleArgs)).To(Succeed(), func() string { - RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", "") - return "Fail to create schedule" - }) + Expect( + veleroutil.VeleroScheduleCreate( + n.Ctx, + n.VeleroCfg.VeleroCLI, + n.VeleroCfg.VeleroNamespace, + n.ScheduleName, + n.ScheduleArgs, + ), + ).To( + Succeed(), + func() string { + veleroutil.RunDebug( + context.Background(), + n.VeleroCfg.VeleroCLI, + n.VeleroCfg.VeleroNamespace, + "", + "", + ) + return "Fail to create schedule" + }) break } } }) By(fmt.Sprintf("Schedule %s is created without any delay\n", n.ScheduleName), func() { - creationTimestamp, err := GetSchedule(n.Ctx, n.VeleroCfg.VeleroNamespace, n.ScheduleName) + creationTimestamp, err := veleroutil.GetSchedule(n.Ctx, n.VeleroCfg.VeleroNamespace, n.ScheduleName) Expect(err).To(Succeed()) creationTime, err := time.Parse(time.RFC3339, strings.Replace(creationTimestamp, "'", "", -1)) @@ -100,7 +136,12 @@ func (n *ScheduleBackup) Backup() error { fmt.Printf("Get backup for #%d time at %v\n", i, now) //Ignore the last minute in the period avoiding met the 1st backup by schedule if i != n.Period-1 { - backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, n.VeleroCfg.VeleroCLI, "default", n.ScheduleName) + backupsInfo, err := veleroutil.GetScheduledBackupsCreationTime( + n.Ctx, + n.VeleroCfg.VeleroCLI, + "default", + n.ScheduleName, + ) Expect(err).To(Succeed()) Expect(backupsInfo).To(BeEmpty()) } @@ -115,10 +156,17 @@ func (n *ScheduleBackup) Backup() error { for i := 0; i < n.verifyTimes; i++ { fmt.Printf("Start to sleep %d minute #%d time...\n", n.Period, i+1) time.Sleep(time.Duration(n.Period) * time.Minute) + bMap := make(map[string]string) - backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, n.VeleroCfg.VeleroCLI, "default", n.ScheduleName) + backupsInfo, err := veleroutil.GetScheduledBackupsCreationTime( + n.Ctx, + n.VeleroCfg.VeleroCLI, + "default", + n.ScheduleName, + ) Expect(err).To(Succeed()) Expect(backupsInfo).To(HaveLen(i + 2)) + for index, bi := range backupsInfo { bList := strings.Split(bi, ",") fmt.Printf("Backup %d: %v\n", index, bList) @@ -126,8 +174,12 @@ func (n *ScheduleBackup) Backup() error { _, err := time.Parse("2006-01-02 15:04:05 -0700 MST", bList[1]) Expect(err).To(Succeed()) } + + randomNum, err := rand.Int(rand.Reader, big.NewInt(int64(len(backupsInfo)))) + Expect(err).To(Succeed()) + if i == n.verifyTimes-1 { - backupInfo := backupsInfo[rand.Intn(len(backupsInfo))] + backupInfo := backupsInfo[randomNum.Int64()] n.randBackupName = strings.Split(backupInfo, ",")[0] } } @@ -141,16 +193,37 @@ func (n *ScheduleBackup) Backup() error { "--wait", } - backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, n.VeleroCfg.VeleroCLI, "default", n.ScheduleName) + backupsInfo, err := veleroutil.GetScheduledBackupsCreationTime( + n.Ctx, + n.VeleroCfg.VeleroCLI, + "default", + n.ScheduleName, + ) Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName)) fmt.Println(backupsInfo) backupCount := len(backupsInfo) By(fmt.Sprintf("Pause schedule %s ......\n", n.ScheduleName), func() { - Expect(VeleroSchedulePause(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed(), func() string { - RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", "") - return "Fail to pause schedule" - }) + Expect( + veleroutil.VeleroSchedulePause( + n.Ctx, + n.VeleroCfg.VeleroCLI, + n.VeleroCfg.VeleroNamespace, + n.ScheduleName, + ), + ).To( + Succeed(), + func() string { + veleroutil.RunDebug( + context.Background(), + n.VeleroCfg.VeleroCLI, + n.VeleroCfg.VeleroNamespace, + "", + "", + ) + return "Fail to pause schedule" + }, + ) }) periodCount := 3 @@ -159,32 +232,61 @@ func (n *ScheduleBackup) Backup() error { time.Sleep(sleepDuration) }) - backupsInfo, err = GetScheduledBackupsCreationTime(n.Ctx, n.VeleroCfg.VeleroCLI, "default", n.ScheduleName) + backupsInfo, err = veleroutil.GetScheduledBackupsCreationTime( + n.Ctx, + n.VeleroCfg.VeleroCLI, + "default", + n.ScheduleName, + ) Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName)) backupCountPostPause := len(backupsInfo) - fmt.Printf("After pause, backkups count is %d\n", backupCountPostPause) + fmt.Printf("After pause, backups count is %d\n", backupCountPostPause) By(fmt.Sprintf("Verify no new backups from %s ......\n", n.ScheduleName), func() { Expect(backupCountPostPause).To(Equal(backupCount)) }) By(fmt.Sprintf("Unpause schedule %s ......\n", n.ScheduleName), func() { - Expect(VeleroScheduleUnpause(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed(), func() string { - RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", "") - return "Fail to unpause schedule" - }) + Expect( + veleroutil.VeleroScheduleUnpause( + n.Ctx, + n.VeleroCfg.VeleroCLI, + n.VeleroCfg.VeleroNamespace, + n.ScheduleName, + ), + ).To( + Succeed(), + func() string { + veleroutil.RunDebug( + context.Background(), + n.VeleroCfg.VeleroCLI, + n.VeleroCfg.VeleroNamespace, + "", + "", + ) + + return "Fail to unpause schedule" + }) }) By(fmt.Sprintf("Sleep for %s ......\n", sleepDuration), func() { time.Sleep(sleepDuration) }) - backupsInfo, err = GetScheduledBackupsCreationTime(n.Ctx, n.VeleroCfg.VeleroCLI, "default", n.ScheduleName) + backupsInfo, err = veleroutil.GetScheduledBackupsCreationTime( + n.Ctx, + n.VeleroCfg.VeleroCLI, + "default", + n.ScheduleName, + ) Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName)) + fmt.Println(backupsInfo) + backupCountPostUnpause := len(backupsInfo) - fmt.Printf("After unpause, backkups count is %d\n", backupCountPostUnpause) + + fmt.Printf("After unpause, backups count is %d\n", backupCountPostUnpause) By(fmt.Sprintf("Verify no new backups by schedule %s ......\n", n.ScheduleName), func() { Expect(backupCountPostUnpause - backupCount).To(BeNumerically(">=", periodCount-1)) }) @@ -194,9 +296,9 @@ func (n *ScheduleBackup) Backup() error { func (n *ScheduleBackup) Verify() error { By("Namespaces were restored", func() { for _, ns := range *n.NSIncluded { - configmap, err := GetConfigmap(n.Client.ClientGo, ns, n.CaseBaseName) - fmt.Printf("Restored configmap is %v\n", configmap) - Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to list configmap in namespace: %q\n", ns)) + cm, err := k8sutil.GetConfigMap(n.Client.ClientGo, ns, n.CaseBaseName) + fmt.Printf("Restored ConfigMap is %v\n", cm) + Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to list CM in namespace: %q\n", ns)) } }) return nil @@ -206,7 +308,15 @@ func (n *ScheduleBackup) Clean() error { if CurrentSpecReport().Failed() && n.VeleroCfg.FailFast { fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } else { - Expect(VeleroScheduleDelete(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed()) + Expect( + veleroutil.VeleroScheduleDelete( + n.Ctx, + n.VeleroCfg.VeleroCLI, + n.VeleroCfg.VeleroNamespace, + n.ScheduleName, + ), + ).To(Succeed()) + Expect(n.TestCase.Clean()).To(Succeed()) } diff --git a/test/util/k8s/configmap.go b/test/util/k8s/configmap.go index 5a49932bae9..5936fcd835a 100644 --- a/test/util/k8s/configmap.go +++ b/test/util/k8s/configmap.go @@ -67,7 +67,7 @@ func WaitForConfigMapComplete(c clientset.Interface, ns, configmapName string) e }) } -func GetConfigmap(c clientset.Interface, ns, secretName string) (*v1.ConfigMap, error) { +func GetConfigMap(c clientset.Interface, ns, secretName string) (*v1.ConfigMap, error) { return c.CoreV1().ConfigMaps(ns).Get(context.TODO(), secretName, metav1.GetOptions{}) } diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 7d6a6d1c493..cc81813bc96 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -250,7 +250,7 @@ func cleanVSpherePluginConfig(c clientset.Interface, ns, secretName, configMapNa } //clear configmap - _, err = k8s.GetConfigmap(c, ns, configMapName) + _, err = k8s.GetConfigMap(c, ns, configMapName) if err == nil { if err := k8s.WaitForConfigmapDelete(c, ns, configMapName); err != nil { return errors.WithMessagef(err, "Failed to clear up vsphere plugin configmap in %s namespace", ns) diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 639fefb015b..3f617c1a552 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -334,7 +334,7 @@ func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st func checkSchedulePhase(ctx context.Context, veleroCLI, veleroNamespace, scheduleName string) error { return wait.PollImmediate(time.Second*5, time.Minute*2, func() (bool, error) { - checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "schedule", "get", scheduleName, "-ojson") + checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "schedule", "get", scheduleName, "-o", "json") jsonBuf, err := common.CMDExecWithOutput(checkCMD) if err != nil { return false, err @@ -354,7 +354,7 @@ func checkSchedulePhase(ctx context.Context, veleroCLI, veleroNamespace, schedul } func checkSchedulePause(ctx context.Context, veleroCLI, veleroNamespace, scheduleName string, pause bool) error { - checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "schedule", "get", scheduleName, "-ojson") + checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "schedule", "get", scheduleName, "-o", "json") jsonBuf, err := common.CMDExecWithOutput(checkCMD) if err != nil { return err @@ -372,7 +372,7 @@ func checkSchedulePause(ctx context.Context, veleroCLI, veleroNamespace, schedul return nil } func CheckScheduleWithResourceOrder(ctx context.Context, veleroCLI, veleroNamespace, scheduleName string, order map[string]string) error { - checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "schedule", "get", scheduleName, "-ojson") + checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "schedule", "get", scheduleName, "-o", "json") jsonBuf, err := common.CMDExecWithOutput(checkCMD) if err != nil { return err @@ -393,8 +393,8 @@ func CheckScheduleWithResourceOrder(ctx context.Context, veleroCLI, veleroNamesp } } -func CheckBackupWithResourceOrder(ctx context.Context, veleroCLI, veleroNamespace, backupName string, order map[string]string) error { - checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "get", "backup", backupName, "-ojson") +func CheckBackupWithResourceOrder(ctx context.Context, veleroCLI, veleroNamespace, backupName string, orderResources map[string]string) error { + checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "get", "backup", backupName, "-o", "json") jsonBuf, err := common.CMDExecWithOutput(checkCMD) if err != nil { return err @@ -407,10 +407,10 @@ func CheckBackupWithResourceOrder(ctx context.Context, veleroCLI, veleroNamespac if backup.Status.Phase != velerov1api.BackupPhaseCompleted { return errors.Errorf("Unexpected restore phase got %s, expecting %s", backup.Status.Phase, velerov1api.BackupPhaseCompleted) } - if reflect.DeepEqual(backup.Spec.OrderedResources, order) { + if reflect.DeepEqual(backup.Spec.OrderedResources, orderResources) { return nil } else { - return fmt.Errorf("resource order %v set in backup command is not equal with order %v stored in backup cr", order, backup.Spec.OrderedResources) + return fmt.Errorf("resource order %v set in backup command is not equal with order %v stored in backup cr", orderResources, backup.Spec.OrderedResources) } } @@ -452,7 +452,7 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin args = append(args, "--snapshot-volumes=false") } // if "--snapshot-volumes" is not provide, snapshot should be taken as default behavior. } else { // DefaultVolumesToFsBackup is false - // Althrough DefaultVolumesToFsBackup is false, but probably DefaultVolumesToFsBackup + // Although DefaultVolumesToFsBackup is false, but probably DefaultVolumesToFsBackup // was set to true in installation CLI in snapshot volume test, so set DefaultVolumesToFsBackup // to false specifically to make sure volume snapshot was taken if backupCfg.UseVolumeSnapshots { @@ -462,7 +462,7 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin args = append(args, "--default-volumes-to-fs-backup=false") } } - // Also Althrough DefaultVolumesToFsBackup is false, but probably DefaultVolumesToFsBackup + // Although DefaultVolumesToFsBackup is false, but probably DefaultVolumesToFsBackup // was set to true in installation CLI in FS volume backup test, so do nothing here, no DefaultVolumesToFsBackup // appear in backup CLI }