From 8a2f2cc349ecf28f261204307cd470c7a6cf791c Mon Sep 17 00:00:00 2001 From: kgarg-px Date: Wed, 22 Nov 2023 07:48:00 +0000 Subject: [PATCH] PB-5402: kdmp can now use koia debug mode Signed-off-by: kgarg-px --- pkg/drivers/kopiabackup/kopiabackup.go | 4 ++- pkg/drivers/kopiadelete/kopiadelete.go | 2 ++ .../kopiamaintenance/kopiamaintenance.go | 2 ++ pkg/drivers/kopiarestore/kopiarestore.go | 4 ++- pkg/drivers/options.go | 9 +++++ pkg/drivers/utils/utils.go | 27 ++++++++++++++ pkg/executor/kopia/kopiabackup.go | 36 +++++++++++++++++++ pkg/executor/kopia/kopiadelete.go | 9 ++++- pkg/executor/kopia/maintenance.go | 10 ++++++ pkg/executor/kopia/restore.go | 9 +++-- pkg/kopia/command.go | 10 ++++++ 11 files changed, 117 insertions(+), 5 deletions(-) diff --git a/pkg/drivers/kopiabackup/kopiabackup.go b/pkg/drivers/kopiabackup/kopiabackup.go index 1c29b6928..5292f7c41 100644 --- a/pkg/drivers/kopiabackup/kopiabackup.go +++ b/pkg/drivers/kopiabackup/kopiabackup.go @@ -287,6 +287,8 @@ func jobFor( "/data", }, " ") + cmd = utils.CheckAndAddKopiaDebugModeAnnotationsCommand(cmd, jobOption) + if jobOption.Compression != "" { splitCmd := strings.Split(cmd, " ") splitCmd = append(splitCmd, "--compression", jobOption.Compression) @@ -507,7 +509,7 @@ func roleFor(live bool) *rbacv1.Role { Rules: []rbacv1.PolicyRule{ { APIGroups: []string{"kdmp.portworx.com"}, - Resources: []string{"volumebackups"}, + Resources: []string{"volumebackups", "dataexports"}, Verbs: []string{rbacv1.VerbAll}, }, }, diff --git a/pkg/drivers/kopiadelete/kopiadelete.go b/pkg/drivers/kopiadelete/kopiadelete.go index 7ee70c20f..4d9ff778c 100644 --- a/pkg/drivers/kopiadelete/kopiadelete.go +++ b/pkg/drivers/kopiadelete/kopiadelete.go @@ -201,6 +201,8 @@ func jobFor( jobOption.VolumeBackupDeleteNamespace, }, " ") + cmd = utils.CheckAndAddKopiaDebugModeAnnotationsCommand(cmd, jobOption) + kopiaExecutorImage, imageRegistrySecret, err := utils.GetExecutorImageAndSecret(drivers.KopiaExecutorImage, jobOption.KopiaImageExecutorSource, jobOption.KopiaImageExecutorSourceNs, diff --git a/pkg/drivers/kopiamaintenance/kopiamaintenance.go b/pkg/drivers/kopiamaintenance/kopiamaintenance.go index 8bb46e4d2..d8a6ddfc5 100644 --- a/pkg/drivers/kopiamaintenance/kopiamaintenance.go +++ b/pkg/drivers/kopiamaintenance/kopiamaintenance.go @@ -204,6 +204,8 @@ func jobFor( jobOption.MaintenanceType, }, " ") + cmd = utils.CheckAndAddKopiaDebugModeAnnotationsCommand(cmd, jobOption) + kopiaExecutorImage, imageRegistrySecret, err := utils.GetExecutorImageAndSecret(drivers.KopiaExecutorImage, jobOption.KopiaImageExecutorSource, jobOption.KopiaImageExecutorSourceNs, diff --git a/pkg/drivers/kopiarestore/kopiarestore.go b/pkg/drivers/kopiarestore/kopiarestore.go index ba0b7c7e3..43b646299 100644 --- a/pkg/drivers/kopiarestore/kopiarestore.go +++ b/pkg/drivers/kopiarestore/kopiarestore.go @@ -206,6 +206,8 @@ func jobFor( vb.Status.SnapshotID, }, " ") + cmd = utils.CheckAndAddKopiaDebugModeAnnotationsCommand(cmd, jobOption) + kopiaExecutorImage, imageRegistrySecret, err := utils.GetExecutorImageAndSecret(drivers.KopiaExecutorImage, jobOption.KopiaImageExecutorSource, jobOption.KopiaImageExecutorSourceNs, @@ -369,7 +371,7 @@ func roleFor() *rbacv1.Role { }, { APIGroups: []string{"kdmp.portworx.com"}, - Resources: []string{"volumebackups"}, + Resources: []string{"volumebackups", "dataexports"}, Verbs: []string{rbacv1.VerbAll}, }, }, diff --git a/pkg/drivers/options.go b/pkg/drivers/options.go index 61d597c86..2cd338679 100644 --- a/pkg/drivers/options.go +++ b/pkg/drivers/options.go @@ -19,6 +19,7 @@ type JobOpts struct { VolumeBackupName string VolumeBackupNamespace string VolumeBackupDeleteName string + KopiaDebugMode bool VolumeBackupDeleteNamespace string DataExportName string SnapshotID string @@ -395,6 +396,14 @@ func WithVolumeBackupDeleteName(name string) JobOption { } } +// WithKopiaDebugMode is job parameter +func WithKopiaDebugMode(debugMode bool) JobOption { + return func(opts *JobOpts) error { + opts.KopiaDebugMode = debugMode + return nil + } +} + // WithVolumeBackupDeleteNamespace is job parameter. func WithVolumeBackupDeleteNamespace(ns string) JobOption { return func(opts *JobOpts) error { diff --git a/pkg/drivers/utils/utils.go b/pkg/drivers/utils/utils.go index 170ab9efd..f94e159b2 100644 --- a/pkg/drivers/utils/utils.go +++ b/pkg/drivers/utils/utils.go @@ -1,6 +1,7 @@ package utils import ( + "context" "errors" "fmt" "os" @@ -12,6 +13,7 @@ import ( storkapi "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1" "github.com/libopenstorage/stork/pkg/k8sutils" "github.com/portworx/kdmp/pkg/drivers" + kdmpops "github.com/portworx/kdmp/pkg/util/ops" "github.com/portworx/kdmp/pkg/version" "github.com/portworx/sched-ops/k8s/apps" "github.com/portworx/sched-ops/k8s/core" @@ -63,6 +65,8 @@ const ( PvcBoundSuccessMsg = "pvc bounded successfully" // PvcBoundFailedMsg pvc not bounded msg PvcBoundFailedMsg = "pvc not bounded" + // KopiaDebugModeEnabled - debug level log messages are enabled for kopia + KopiaDebugModeEnabled = "kopia-debug-mode" ) var ( @@ -876,3 +880,26 @@ func IsJobPodMountFailed(job *batchv1.Job, namespace string) bool { } return false } + +func IsKopiaDebugModeAnnotationsEnabled(jobOption drivers.JobOpts) bool { + dataExportCR, err := kdmpops.Instance().GetDataExport(context.Background(), jobOption.DataExportName, jobOption.Namespace) + if err != nil { + logrus.Tracef("error reading data export job: %v", err) + return false + } + + if _, ok := dataExportCR.Annotations[KopiaDebugModeEnabled]; ok { + logrus.Infof("annotation %v found in the data export CR.", KopiaDebugModeEnabled) + return true + } + return false +} + +func CheckAndAddKopiaDebugModeAnnotationsCommand(cmd string, jobOption drivers.JobOpts) string { + if IsKopiaDebugModeAnnotationsEnabled(jobOption) { + splitCmd := strings.Split(cmd, " ") + splitCmd = append(splitCmd, "--log-level", "debug") + cmd = strings.Join(splitCmd, " ") + } + return cmd +} diff --git a/pkg/executor/kopia/kopiabackup.go b/pkg/executor/kopia/kopiabackup.go index 4bbf425e4..c2d44cb77 100644 --- a/pkg/executor/kopia/kopiabackup.go +++ b/pkg/executor/kopia/kopiabackup.go @@ -35,6 +35,7 @@ var ( bkpNamespace string compression string excludeFileList string + logLevelDebug string ) var ( @@ -69,6 +70,7 @@ func newBackupCommand() *cobra.Command { backupCommand.Flags().StringVar(&sourcePathGlob, "source-path-glob", "", "The regexp should match only one path that will be used for backup") backupCommand.Flags().StringVar(&compression, "compression", "", "Compression type to be used") backupCommand.Flags().StringVar(&excludeFileList, "exclude-file-list", "", " list of dir names that need to be exclude in the kopia snapshot") + backupCommand.Flags().StringVar(&logLevelDebug, "log-level", "", "If debug mode in kopia is to be used") return backupCommand } @@ -249,6 +251,11 @@ func runKopiaCreateRepo(repository *executor.Repository) error { if err != nil { return err } + + // Now check if debug log level is to be added in the command which got prepared above. + // check if debug level exists + repoCreateCmd = isKopiaDebugModeEnabled(repoCreateCmd, logLevelDebug) + // NFS doesn't need any special treatment for repo create command // hence no case exist for it. switch repository.Type { @@ -323,6 +330,9 @@ func runKopiaBackup(repository *executor.Repository, sourcePath string) error { if err != nil { return err } + + // Check and add debug level logs for kopia backup command + backupCmd = isKopiaDebugModeEnabled(backupCmd, logLevelDebug) // This is needed to handle case where after kopia repo create was successful and // the pod got terminated. Now user triggers another backup, so we need to pass // credentials for "snapshot create". @@ -386,6 +396,8 @@ func runKopiaRepositoryConnect(repository *executor.Repository) error { if err != nil { return err } + // Check and add debug level logs for kopia connect command + connectCmd = isKopiaDebugModeEnabled(connectCmd, logLevelDebug) switch repository.Type { case storkv1.BackupLocationS3: @@ -427,6 +439,8 @@ func runKopiaRepositoryConnect(repository *executor.Repository) error { func setGlobalPolicy() error { logrus.Infof("Setting global policy") policyCmd, err := kopia.SetGlobalPolicyCommand() + policyCmd = isKopiaDebugModeEnabled(policyCmd, logLevelDebug) + if err != nil { return err } @@ -483,6 +497,10 @@ func runKopiaExcludeFileList(repository *executor.Repository, sourcePath string) if err != nil { return err } + + // Check and add debug level logs for kopia exclude file list command + excludeFileListCmd = isKopiaDebugModeEnabled(excludeFileListCmd, logLevelDebug) + excludeFileListExecutor := kopia.NewExcludeFileListExecutor(excludeFileListCmd) if err := excludeFileListExecutor.Run(); err != nil { err = fmt.Errorf("failed to run exclude file list command: %v", err) @@ -529,6 +547,10 @@ func runKopiaCompression(repository *executor.Repository, sourcePath string) err if err != nil { return err } + + // Check and add debug level logs for kopia compression command + compressionCmd = isKopiaDebugModeEnabled(compressionCmd, logLevelDebug) + compressionExecutor := kopia.NewCompressionExecutor(compressionCmd) if err := compressionExecutor.Run(); err != nil { err = fmt.Errorf("failed to run compression command: %v", err) @@ -677,3 +699,17 @@ func addPolicySetting(policyCmd *kopia.Command) *kopia.Command { return policyCmd } + +func isKopiaDebugModeEnabled(kopiaCommand *kopia.Command, logLevelDebug string) *kopia.Command { + if logLevelDebug != "" { + kopiaCommand = addLogLevelDebugToCommand(kopiaCommand) + } + return kopiaCommand +} + +func addLogLevelDebugToCommand(initCmd *kopia.Command) *kopia.Command { + // Kopia command to be run with debug log levels now. + initCmd.AddArg("--log-level") + initCmd.AddArg("debug") + return initCmd +} diff --git a/pkg/executor/kopia/kopiadelete.go b/pkg/executor/kopia/kopiadelete.go index ac4076cb8..6639227fe 100644 --- a/pkg/executor/kopia/kopiadelete.go +++ b/pkg/executor/kopia/kopiadelete.go @@ -21,6 +21,7 @@ func newDeleteCommand() *cobra.Command { credSecretNamespace string volumeBackupDeleteName string volumeBackupDeleteNamespace string + logLevelDebug string ) deleteCommand := &cobra.Command{ Use: "delete", @@ -34,6 +35,7 @@ func newDeleteCommand() *cobra.Command { deleteCommand.Flags().StringVar(&credSecretNamespace, "cred-secret-namespace", "", "cred secret namespace for kopia backup snapshot that need to be deleted") deleteCommand.Flags().StringVar(&volumeBackupDeleteName, "volume-backup-delete-name", "", "volumeBackupdelete CR name for kopia backup snapshot that need to be deleted") deleteCommand.Flags().StringVar(&volumeBackupDeleteNamespace, "volume-backup-delete-namespace", "", "volumeBackupdelete CR namespace for kopia backup snapshot that need to be deleted") + deleteCommand.Flags().StringVar(&logLevelDebug, "log-level", "", "If debug mode in kopia is to be used") return deleteCommand } @@ -133,6 +135,9 @@ func runKopiaDelete(repository *executor.Repository, snapshotID string) error { logrus.Errorf("%s %v", fn, errMsg) return fmt.Errorf(errMsg) } + // Check and add debug level logs for kopia delete command + deleteCmd = isKopiaDebugModeEnabled(deleteCmd, logLevelDebug) + initExecutor := kopia.NewDeleteExecutor(deleteCmd) if err := initExecutor.Run(); err != nil { errMsg := fmt.Sprintf("running delete backup snapshot command for snapshotID [%v] failed: %v", snapshotID, err) @@ -163,11 +168,13 @@ func runKopiaSnapshotList(repository *executor.Repository) ([]string, error) { var listCmd *kopia.Command logrus.Infof("Executing kopia snapshot list command") listCmd, err = kopia.GetListCommand() - if err != nil { return nil, err } + // Check and add bebug level logs for kopia snapshot list command + listCmd = isKopiaDebugModeEnabled(listCmd, logLevelDebug) + listExecutor := kopia.NewListExecutor(listCmd) if err := listExecutor.Run(); err != nil { err = fmt.Errorf("failed to run snapshot list command: %v", err) diff --git a/pkg/executor/kopia/maintenance.go b/pkg/executor/kopia/maintenance.go index b5e2ffd12..e2586df36 100644 --- a/pkg/executor/kopia/maintenance.go +++ b/pkg/executor/kopia/maintenance.go @@ -38,6 +38,7 @@ func newMaintenanceCommand() *cobra.Command { credSecretName string credSecretNamespace string maintenanceType string + logLevelDebug string ) maintenanceCommand := &cobra.Command{ Use: "maintenance", @@ -51,6 +52,7 @@ func newMaintenanceCommand() *cobra.Command { maintenanceCommand.Flags().StringVar(&maintenanceStatusName, "maintenance-status-name", "", "backuplocation maintenance status CR name, where repo maintenance status will be stored") maintenanceCommand.Flags().StringVar(&maintenanceStatusNamespace, "maintenance-status-namespace", "", "backuplocation maintenance status CR namespace, where repo maintenance status will be stored") maintenanceCommand.Flags().StringVar(&maintenanceType, "maintenance-type", "", "full - will run full maintenance and quick - will run quick maintenance") + maintenanceCommand.Flags().StringVar(&logLevelDebug, "log-level", "", "If debug mode in kopia is to be used") return maintenanceCommand } @@ -281,6 +283,8 @@ func runKopiaQuickMaintenanceExecute(repository *executor.Repository) error { return fmt.Errorf(errMsg) } + // Check and add debug log level for kopia maintenance command + maintenanceRunCmd = isKopiaDebugModeEnabled(maintenanceRunCmd, logLevelDebug) initExecutor := kopia.NewMaintenanceRunExecutor(maintenanceRunCmd) if err := initExecutor.Run(); err != nil { errMsg := fmt.Sprintf("running maintenance run command for [%v] failed: %v", repository.Name, err) @@ -312,6 +316,8 @@ func runKopiaMaintenanceExecute(repository *executor.Repository) error { logrus.Errorf("%s %v", fn, errMsg) return fmt.Errorf(errMsg) } + // Check and add debug log level for kopia maintenance command + maintenanceRunCmd = isKopiaDebugModeEnabled(maintenanceRunCmd, logLevelDebug) initExecutor := kopia.NewMaintenanceRunExecutor(maintenanceRunCmd) if err := initExecutor.Run(); err != nil { errMsg := fmt.Sprintf("running maintenance run command for [%v] failed: %v", repository.Name, err) @@ -343,6 +349,10 @@ func runKopiaMaintenanceSet(repository *executor.Repository) error { logrus.Errorf("%s %v", fn, errMsg) return fmt.Errorf(errMsg) } + + // Check and add debug log level for kopia maintenance set command + maintenanceSetCmd = isKopiaDebugModeEnabled(maintenanceSetCmd, logLevelDebug) + initExecutor := kopia.NewMaintenanceSetExecutor(maintenanceSetCmd) if err := initExecutor.Run(); err != nil { errMsg := fmt.Sprintf("running maintenance set command for failed: %v", err) diff --git a/pkg/executor/kopia/restore.go b/pkg/executor/kopia/restore.go index f5be6e2eb..16f21d9d3 100644 --- a/pkg/executor/kopia/restore.go +++ b/pkg/executor/kopia/restore.go @@ -18,8 +18,9 @@ var ( func newRestoreCommand() *cobra.Command { var ( - targetPath string - snapshotID string + targetPath string + snapshotID string + logLevelDebug string ) restoreCommand := &cobra.Command{ Use: "restore", @@ -40,6 +41,7 @@ func newRestoreCommand() *cobra.Command { restoreCommand.Flags().StringVar(&targetPath, "target-path", "", "Destination path for kopia restore") restoreCommand.Flags().StringVar(&snapshotID, "snapshot-id", "", "Snapshot id of the restore") restoreCommand.Flags().StringVar(&appRestoreCR, "app-restore-cr", "", "ApplicationRestore CR name") + restoreCommand.Flags().StringVar(&logLevelDebug, "log-level", "", "If debug mode in kopia is to be used") return restoreCommand } @@ -108,6 +110,9 @@ func runKopiaRestore(repository *executor.Repository, targetPath, snapshotID str return err } + // Check and add debug level logs for kopia restore command + restoreCmd = isKopiaDebugModeEnabled(restoreCmd, logLevelDebug) + initExecutor := kopia.NewRestoreExecutor(restoreCmd) if err := initExecutor.Run(); err != nil { err = fmt.Errorf("failed to run restore command: %v", err) diff --git a/pkg/kopia/command.go b/pkg/kopia/command.go index e453733e9..c383c728e 100644 --- a/pkg/kopia/command.go +++ b/pkg/kopia/command.go @@ -191,6 +191,7 @@ func (c *Command) BackupCmd() *exec.Cmd { configFile, "--json", } + argsSlice = append(argsSlice, c.Flags...) // Get the cmd args argsSlice = append(argsSlice, c.Args...) @@ -307,6 +308,7 @@ func (c *Command) RestoreCmd() *exec.Cmd { "--config-file", configFile, } + argsSlice = append(argsSlice, c.Flags...) // Get the cmd args argsSlice = append(argsSlice, c.Args...) @@ -330,6 +332,7 @@ func (c *Command) SetPolicyCmd() *exec.Cmd { configFile, "--global", } + argsSlice = append(argsSlice, c.Flags...) // Get the cmd args argsSlice = append(argsSlice, c.Args...) @@ -355,6 +358,7 @@ func (c *Command) DeleteCmd() *exec.Cmd { configFile, "--delete", } + argsSlice = append(argsSlice, c.Flags...) // Get the cmd args argsSlice = append(argsSlice, c.Args...) @@ -379,6 +383,7 @@ func (c *Command) QuickMaintenanceRunCmd() *exec.Cmd { "--config-file", configFile, } + argsSlice = append(argsSlice, c.Flags...) // Get the cmd args argsSlice = append(argsSlice, c.Args...) @@ -403,6 +408,7 @@ func (c *Command) MaintenanceRunCmd() *exec.Cmd { configFile, "--full", } + argsSlice = append(argsSlice, c.Flags...) // Get the cmd args argsSlice = append(argsSlice, c.Args...) @@ -427,6 +433,7 @@ func (c *Command) SnapshotListCmd() *exec.Cmd { "--config-file", configFile, } + argsSlice = append(argsSlice, c.Flags...) // Get the cmd args argsSlice = append(argsSlice, c.Args...) @@ -453,6 +460,7 @@ func (c *Command) MaintenanceSetCmd() *exec.Cmd { "--config-file", configFile, } + argsSlice = append(argsSlice, c.Flags...) // Get the cmd args argsSlice = append(argsSlice, c.Args...) @@ -479,6 +487,7 @@ func (c *Command) CompressionCmd() *exec.Cmd { "--config-file", configFile, } + argsSlice = append(argsSlice, c.Flags...) // Get the cmd args argsSlice = append(argsSlice, c.Args...) @@ -503,6 +512,7 @@ func (c *Command) ExcludeFileListCmd() *exec.Cmd { "--config-file", configFile, } + commaSplit := strings.Split(c.ExcludeFileList, ",") for _, file := range commaSplit { argsSlice = append(argsSlice, "--add-ignore")