PB-4844: kdmp can now use koia debug mode

Signed-off-by: kgarg-px <[email protected]>

pkg/drivers/kopiabackup/kopiabackup.go

@@ -287,6 +287,8 @@ func jobFor(
 		"/data",
 	}, " ")
 
+	cmd = utils.CheckAndAddKopiaDebugModeEnabled(cmd, jobOption)
+
 	if jobOption.Compression != "" {
 		splitCmd := strings.Split(cmd, " ")
 		splitCmd = append(splitCmd, "--compression", jobOption.Compression)
@@ -507,7 +509,7 @@ func roleFor(live bool) *rbacv1.Role {
 		Rules: []rbacv1.PolicyRule{
 			{
 				APIGroups: []string{"kdmp.portworx.com"},
-				Resources: []string{"volumebackups"},
+				Resources: []string{"volumebackups", "dataexports"},
 				Verbs:     []string{rbacv1.VerbAll},
 			},
 		},

pkg/drivers/kopiadelete/kopiadelete.go

@@ -201,6 +201,8 @@ func jobFor(
 		jobOption.VolumeBackupDeleteNamespace,
 	}, " ")
 
+	cmd = utils.CheckAndAddKopiaDebugModeEnabled(cmd, jobOption)
+
 	kopiaExecutorImage, imageRegistrySecret, err := utils.GetExecutorImageAndSecret(drivers.KopiaExecutorImage,
 		jobOption.KopiaImageExecutorSource,
 		jobOption.KopiaImageExecutorSourceNs,

pkg/drivers/kopiamaintenance/kopiamaintenance.go

@@ -204,6 +204,8 @@ func jobFor(
 		jobOption.MaintenanceType,
 	}, " ")
 
+	cmd = utils.CheckAndAddKopiaDebugModeEnabled(cmd, jobOption)
+
 	kopiaExecutorImage, imageRegistrySecret, err := utils.GetExecutorImageAndSecret(drivers.KopiaExecutorImage,
 		jobOption.KopiaImageExecutorSource,
 		jobOption.KopiaImageExecutorSourceNs,

pkg/drivers/kopiarestore/kopiarestore.go

@@ -206,6 +206,8 @@ func jobFor(
 		vb.Status.SnapshotID,
 	}, " ")
 
+	cmd = utils.CheckAndAddKopiaDebugModeEnabled(cmd, jobOption)
+
 	kopiaExecutorImage, imageRegistrySecret, err := utils.GetExecutorImageAndSecret(drivers.KopiaExecutorImage,
 		jobOption.KopiaImageExecutorSource,
 		jobOption.KopiaImageExecutorSourceNs,
@@ -369,7 +371,7 @@ func roleFor() *rbacv1.Role {
 			},
 			{
 				APIGroups: []string{"kdmp.portworx.com"},
-				Resources: []string{"volumebackups"},
+				Resources: []string{"volumebackups", "dataexports"},
 				Verbs:     []string{rbacv1.VerbAll},
 			},
 		},

pkg/drivers/options.go

@@ -19,6 +19,7 @@ type JobOpts struct {
 	VolumeBackupName            string
 	VolumeBackupNamespace       string
 	VolumeBackupDeleteName      string
+	KopiaDebugMode              bool
 	VolumeBackupDeleteNamespace string
 	DataExportName              string
 	SnapshotID                  string
@@ -395,6 +396,14 @@ func WithVolumeBackupDeleteName(name string) JobOption {
 	}
 }
 
+// WithKopiaDebugMode is job parameter
+func WithKopiaDebugMode(debugMode bool) JobOption {
+	return func(opts *JobOpts) error {
+		opts.KopiaDebugMode = debugMode
+		return nil
+	}
+}
+
 // WithVolumeBackupDeleteNamespace is job parameter.
 func WithVolumeBackupDeleteNamespace(ns string) JobOption {
 	return func(opts *JobOpts) error {

pkg/drivers/utils/utils.go

@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"os"
@@ -12,6 +13,7 @@ import (
 	storkapi "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1"
 	"github.com/libopenstorage/stork/pkg/k8sutils"
 	"github.com/portworx/kdmp/pkg/drivers"
+	kdmpops "github.com/portworx/kdmp/pkg/util/ops"
 	"github.com/portworx/kdmp/pkg/version"
 	"github.com/portworx/sched-ops/k8s/apps"
 	"github.com/portworx/sched-ops/k8s/core"
@@ -63,6 +65,8 @@ const (
 	PvcBoundSuccessMsg = "pvc bounded successfully"
 	// PvcBoundFailedMsg pvc not bounded msg
 	PvcBoundFailedMsg = "pvc not bounded"
+	// KopiaDebugModeEnabled - debug level log messages are enabled for kopia
+	KopiaDebugModeEnabled = "kopia-debug-mode"
 )
 
 var (
@@ -876,3 +880,37 @@ func IsJobPodMountFailed(job *batchv1.Job, namespace string) bool {
 	}
 	return false
 }
+
+func IsKopiaDebugModeAnnotationsEnabled(jobOption drivers.JobOpts) bool {
+	// Check for kopia delete and maintenance operations. Since, the debug is passed from job options
+	// and not from Data Export CR.
+	if jobOption.KopiaDebugMode {
+		return true
+	}
+
+	dataExportCR, err := kdmpops.Instance().GetDataExport(context.Background(), jobOption.DataExportName, jobOption.Namespace)
+	if err != nil {
+		logrus.Tracef("error reading data export job: %v", err)
+		return false
+	}
+
+	if _, ok := dataExportCR.Annotations[KopiaDebugModeEnabled]; ok {
+		logrus.Infof("annotation %v found in the data export CR.", KopiaDebugModeEnabled)
+		return true
+	}
+	return false
+}
+
+func CheckAndAddKopiaDebugModeEnabled(cmd string, jobOption drivers.JobOpts) string {
+	if IsKopiaDebugModeAnnotationsEnabled(jobOption) {
+		cmd = AddKopiaDebugModeCommand(cmd)
+	}
+	return cmd
+}
+
+func AddKopiaDebugModeCommand(cmd string) string {
+	splitCmd := strings.Split(cmd, " ")
+	splitCmd = append(splitCmd, "--log-level", "debug")
+	cmd = strings.Join(splitCmd, " ")
+	return cmd
+}

pkg/executor/kopia/kopiabackup.go

@@ -35,6 +35,7 @@ var (
 	bkpNamespace    string
 	compression     string
 	excludeFileList string
+	logLevelDebug   string
 )
 
 var (
@@ -69,6 +70,7 @@ func newBackupCommand() *cobra.Command {
 	backupCommand.Flags().StringVar(&sourcePathGlob, "source-path-glob", "", "The regexp should match only one path that will be used for backup")
 	backupCommand.Flags().StringVar(&compression, "compression", "", "Compression type to be used")
 	backupCommand.Flags().StringVar(&excludeFileList, "exclude-file-list", "", " list of dir names that need to be exclude in the kopia snapshot")
+	backupCommand.Flags().StringVar(&logLevelDebug, "log-level", "", "If debug mode in kopia is to be used")
 
 	return backupCommand
 }
@@ -249,6 +251,9 @@ func runKopiaCreateRepo(repository *executor.Repository) error {
 	if err != nil {
 		return err
 	}
+	// Now check if debug log level is to be added in the command which got prepared above.
+	// check if debug level exists
+	repoCreateCmd = addLogLevelDebugToCommand(repoCreateCmd, logLevelDebug)
 	// NFS doesn't need any special treatment for repo create command
 	// hence no case exist for it.
 	switch repository.Type {
@@ -323,6 +328,8 @@ func runKopiaBackup(repository *executor.Repository, sourcePath string) error {
 	if err != nil {
 		return err
 	}
+	// Check and add debug level logs for kopia backup command
+	backupCmd = addLogLevelDebugToCommand(backupCmd, logLevelDebug)
 	// This is needed to handle case where after kopia repo create was successful and
 	// the pod got terminated. Now user triggers another backup, so we need to pass
 	// credentials for "snapshot create".
@@ -386,6 +393,8 @@ func runKopiaRepositoryConnect(repository *executor.Repository) error {
 	if err != nil {
 		return err
 	}
+	// Check and add debug level logs for kopia connect command
+	connectCmd = addLogLevelDebugToCommand(connectCmd, logLevelDebug)
 
 	switch repository.Type {
 	case storkv1.BackupLocationS3:
@@ -430,6 +439,7 @@ func setGlobalPolicy() error {
 	if err != nil {
 		return err
 	}
+	policyCmd = addLogLevelDebugToCommand(policyCmd, logLevelDebug)
 	// As we don't want kopia maintenance to kick in and trigger global policy on default values
 	// for the repository, setting them to very high values
 	policyCmd = addPolicySetting(policyCmd)
@@ -483,6 +493,8 @@ func runKopiaExcludeFileList(repository *executor.Repository, sourcePath string)
 	if err != nil {
 		return err
 	}
+	// Check and add debug level logs for kopia exclude file list command
+	excludeFileListCmd = addLogLevelDebugToCommand(excludeFileListCmd, logLevelDebug)
 	excludeFileListExecutor := kopia.NewExcludeFileListExecutor(excludeFileListCmd)
 	if err := excludeFileListExecutor.Run(); err != nil {
 		err = fmt.Errorf("failed to run exclude file list command: %v", err)
@@ -529,6 +541,8 @@ func runKopiaCompression(repository *executor.Repository, sourcePath string) err
 	if err != nil {
 		return err
 	}
+	// Check and add debug level logs for kopia compression command
+	compressionCmd = addLogLevelDebugToCommand(compressionCmd, logLevelDebug)
 	compressionExecutor := kopia.NewCompressionExecutor(compressionCmd)
 	if err := compressionExecutor.Run(); err != nil {
 		err = fmt.Errorf("failed to run compression command: %v", err)
@@ -677,3 +691,12 @@ func addPolicySetting(policyCmd *kopia.Command) *kopia.Command {
 
 	return policyCmd
 }
+
+func addLogLevelDebugToCommand(kopiaCommand *kopia.Command, logLevelDebug string) *kopia.Command {
+	if logLevelDebug != "" {
+		// Kopia command to be run with debug log levels now.
+		kopiaCommand.AddArg("--log-level")
+		kopiaCommand.AddArg("debug")
+	}
+	return kopiaCommand
+}

pkg/executor/kopia/kopiadelete.go

@@ -21,6 +21,7 @@ func newDeleteCommand() *cobra.Command {
 		credSecretNamespace         string
 		volumeBackupDeleteName      string
 		volumeBackupDeleteNamespace string
+		logLevelDebug               string
 	)
 	deleteCommand := &cobra.Command{
 		Use:   "delete",
@@ -34,6 +35,7 @@ func newDeleteCommand() *cobra.Command {
 	deleteCommand.Flags().StringVar(&credSecretNamespace, "cred-secret-namespace", "", "cred secret namespace for kopia backup snapshot that need to be deleted")
 	deleteCommand.Flags().StringVar(&volumeBackupDeleteName, "volume-backup-delete-name", "", "volumeBackupdelete CR name for kopia backup snapshot that need to be deleted")
 	deleteCommand.Flags().StringVar(&volumeBackupDeleteNamespace, "volume-backup-delete-namespace", "", "volumeBackupdelete CR namespace for kopia backup snapshot that need to be deleted")
+	deleteCommand.Flags().StringVar(&logLevelDebug, "log-level", "", "If debug mode in kopia is to be used")
 	return deleteCommand
 }
 
@@ -133,6 +135,8 @@ func runKopiaDelete(repository *executor.Repository, snapshotID string) error {
 		logrus.Errorf("%s %v", fn, errMsg)
 		return fmt.Errorf(errMsg)
 	}
+	// Check and add debug level logs for kopia delete command
+	deleteCmd = addLogLevelDebugToCommand(deleteCmd, logLevelDebug)
 	initExecutor := kopia.NewDeleteExecutor(deleteCmd)
 	if err := initExecutor.Run(); err != nil {
 		errMsg := fmt.Sprintf("running delete backup snapshot command for snapshotID [%v] failed: %v", snapshotID, err)
@@ -163,11 +167,11 @@ func runKopiaSnapshotList(repository *executor.Repository) ([]string, error) {
 	var listCmd *kopia.Command
 	logrus.Infof("Executing kopia snapshot list command")
 	listCmd, err = kopia.GetListCommand()
-
 	if err != nil {
 		return nil, err
 	}
-
+	// Check and add bebug level logs for kopia snapshot list command
+	listCmd = addLogLevelDebugToCommand(listCmd, logLevelDebug)
 	listExecutor := kopia.NewListExecutor(listCmd)
 	if err := listExecutor.Run(); err != nil {
 		err = fmt.Errorf("failed to run snapshot list command: %v", err)

pkg/executor/kopia/maintenance.go

@@ -38,6 +38,7 @@ func newMaintenanceCommand() *cobra.Command {
 		credSecretName      string
 		credSecretNamespace string
 		maintenanceType     string
+		logLevelDebug       string
 	)
 	maintenanceCommand := &cobra.Command{
 		Use:   "maintenance",
@@ -51,6 +52,7 @@ func newMaintenanceCommand() *cobra.Command {
 	maintenanceCommand.Flags().StringVar(&maintenanceStatusName, "maintenance-status-name", "", "backuplocation maintenance status CR name, where repo maintenance status will be stored")
 	maintenanceCommand.Flags().StringVar(&maintenanceStatusNamespace, "maintenance-status-namespace", "", "backuplocation maintenance status CR namespace, where repo maintenance status will be stored")
 	maintenanceCommand.Flags().StringVar(&maintenanceType, "maintenance-type", "", "full - will run full maintenance and quick - will run quick maintenance")
+	maintenanceCommand.Flags().StringVar(&logLevelDebug, "log-level", "", "If debug mode in kopia is to be used")
 	return maintenanceCommand
 }
 
@@ -280,7 +282,8 @@ func runKopiaQuickMaintenanceExecute(repository *executor.Repository) error {
 		logrus.Errorf("%s %v", fn, errMsg)
 		return fmt.Errorf(errMsg)
 	}
-
+	// Check and add debug log level for kopia maintenance command
+	maintenanceRunCmd = addLogLevelDebugToCommand(maintenanceRunCmd, logLevelDebug)
 	initExecutor := kopia.NewMaintenanceRunExecutor(maintenanceRunCmd)
 	if err := initExecutor.Run(); err != nil {
 		errMsg := fmt.Sprintf("running maintenance run command for [%v] failed: %v", repository.Name, err)
@@ -312,6 +315,8 @@ func runKopiaMaintenanceExecute(repository *executor.Repository) error {
 		logrus.Errorf("%s %v", fn, errMsg)
 		return fmt.Errorf(errMsg)
 	}
+	// Check and add debug log level for kopia maintenance command
+	maintenanceRunCmd = addLogLevelDebugToCommand(maintenanceRunCmd, logLevelDebug)
 	initExecutor := kopia.NewMaintenanceRunExecutor(maintenanceRunCmd)
 	if err := initExecutor.Run(); err != nil {
 		errMsg := fmt.Sprintf("running maintenance run command for [%v] failed: %v", repository.Name, err)
@@ -343,6 +348,8 @@ func runKopiaMaintenanceSet(repository *executor.Repository) error {
 		logrus.Errorf("%s %v", fn, errMsg)
 		return fmt.Errorf(errMsg)
 	}
+	// Check and add debug log level for kopia maintenance set command
+	maintenanceSetCmd = addLogLevelDebugToCommand(maintenanceSetCmd, logLevelDebug)
 	initExecutor := kopia.NewMaintenanceSetExecutor(maintenanceSetCmd)
 	if err := initExecutor.Run(); err != nil {
 		errMsg := fmt.Sprintf("running maintenance set command for failed: %v", err)

pkg/executor/kopia/restore.go

@@ -18,8 +18,9 @@ var (
 
 func newRestoreCommand() *cobra.Command {
 	var (
-		targetPath string
-		snapshotID string
+		targetPath    string
+		snapshotID    string
+		logLevelDebug string
 	)
 	restoreCommand := &cobra.Command{
 		Use:   "restore",
@@ -40,6 +41,7 @@ func newRestoreCommand() *cobra.Command {
 	restoreCommand.Flags().StringVar(&targetPath, "target-path", "", "Destination path for kopia restore")
 	restoreCommand.Flags().StringVar(&snapshotID, "snapshot-id", "", "Snapshot id of the restore")
 	restoreCommand.Flags().StringVar(&appRestoreCR, "app-restore-cr", "", "ApplicationRestore CR name")
+	restoreCommand.Flags().StringVar(&logLevelDebug, "log-level", "", "If debug mode in kopia is to be used")
 
 	return restoreCommand
 }
@@ -107,7 +109,8 @@ func runKopiaRestore(repository *executor.Repository, targetPath, snapshotID str
 	if err != nil {
 		return err
 	}
-
+	// Check and add debug level logs for kopia restore command
+	restoreCmd = addLogLevelDebugToCommand(restoreCmd, logLevelDebug)
 	initExecutor := kopia.NewRestoreExecutor(restoreCmd)
 	if err := initExecutor.Run(); err != nil {
 		err = fmt.Errorf("failed to run restore command: %v", err)