pb-4872: Added change to support ignore file option for kdmp snapshot.

pkg/controllers/dataexport/reconcile.go

@@ -62,6 +62,7 @@ const (
 	pxbackupAnnotationCreateByValue = "px-backup"
 	backupObjectUIDKey              = kdmpAnnotationPrefix + "backupobject-uid"
 	pvcUIDKey                       = kdmpAnnotationPrefix + "pvc-uid"
+	kdmpStorageClassKey             = kdmpAnnotationPrefix + "storage-class"
 	volumeSnapShotCRDirectory       = "csi-generic"
 	snapDeleteAnnotation            = "snapshotScheduledForDeletion"
 	snapRestoreAnnotation           = "snapshotScheduledForRestore"
@@ -76,6 +77,7 @@ const (
 	defaultTimeout        = 1 * time.Minute
 	progressCheckInterval = 5 * time.Second
 	compressionKey        = "KDMP_COMPRESSION"
+	excludeFileListKey    = "KDMP_EXCLUDE_FILE_LIST"
 	backupPath            = "KDMP_BACKUP_PATH"
 )
 
@@ -261,6 +263,8 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er
 
 		var compressionType string
 		var podDataPath string
+		var excludeFileList string
+		pvcStorageClass := dataExport.Labels[kdmpStorageClassKey]
 		var backupLocation *storkapi.BackupLocation
 		var data updateDataExportDetail
 		if driverName != drivers.Rsync {
@@ -273,11 +277,32 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er
 			kdmpData, err := core.Instance().GetConfigMap(utils.KdmpConfigmapName, utils.KdmpConfigmapNamespace)
 			if err != nil {
 				logrus.Errorf("failed reading config map %v: %v", utils.KdmpConfigmapName, err)
-				logrus.Warnf("default to %s compression", utils.DefaultCompresion)
+				if err != nil {
+					msg := fmt.Sprintf("Failed in parsing the excludeFileList configmap parameter from configmap [%v/%v]: %v", utils.KdmpConfigmapNamespace, utils.KdmpConfigmapName, err)
+					logrus.Errorf(msg)
+					data := updateDataExportDetail{
+						status: kdmpapi.DataExportStatusFailed,
+						reason: msg,
+					}
+					return false, c.updateStatus(dataExport, data)
+				}
+			}
+			compressionType = kdmpData.Data[compressionKey]
+			if len(compressionType) == 0 {
 				compressionType = utils.DefaultCompresion
-			} else {
-				compressionType = kdmpData.Data[compressionKey]
-				podDataPath = kdmpData.Data[backupPath]
+			}
+			podDataPath = kdmpData.Data[backupPath]
+			if len(kdmpData.Data[excludeFileListKey]) != 0 {
+				excludeFileList, err = parseExcludeFileListKey(pvcStorageClass, kdmpData.Data[excludeFileListKey])
+				if err != nil {
+					msg := fmt.Sprintf("Failed in parsing the excludeFileList configmap parameter from configmap [%v/%v]", utils.KdmpConfigmapNamespace, utils.KdmpConfigmapName)
+					logrus.Errorf(msg)
+					data := updateDataExportDetail{
+						status: kdmpapi.DataExportStatusFailed,
+						reason: msg,
+					}
+					return false, c.updateStatus(dataExport, data)
+				}
 			}
 			blName := dataExport.Spec.Destination.Name
 			blNamespace := dataExport.Spec.Destination.Namespace
@@ -308,6 +333,7 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er
 			driver,
 			srcPVCName,
 			compressionType,
+			excludeFileList,
 			dataExport,
 			podDataPath,
 			utils.KdmpConfigmapName,
@@ -503,6 +529,35 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er
 	return false, nil
 }
 
+// If pvc storageclass and the configured storageclass matches, extract the configured ignore file list and return it.
+// For code reference, adding the sample of the configmap parameter.
+//
+//	KDMP_EXCLUDE_FILE_LIST: |
+//	   px-db=dir1,file1,dir2
+//	   mysql=dir1,file1,dir2
+//
+// After we read, the parameter value comes as follow:
+// "px-db=dir1,file1,dir2\nmysql=dir1,file1,dir2\n"
+func parseExcludeFileListKey(pvcStorageClass string, excludeFileListValue string) (string, error) {
+	// trim the ending "\n" character, if it present
+	excludeFileListValue = strings.TrimSuffix(string(excludeFileListValue), "\n")
+	storageClassList := strings.Split(excludeFileListValue, "\n")
+	var excludeFileList string
+	for _, storageClass := range storageClassList {
+		equalSignSplit := strings.Split(storageClass, "=")
+		if len(equalSignSplit) != 2 {
+			return "", fmt.Errorf("invalid exclude file list in the configmap parameter")
+		}
+		// if the PVC storageclass and configure storageclass are same, extract the configured ignore file list
+		if pvcStorageClass == equalSignSplit[0] {
+			excludeFileList = equalSignSplit[1]
+		}
+
+	}
+	logrus.Infof("parseExcludeFileListKey: configured excludeFileList - %v", excludeFileList)
+	return excludeFileList, nil
+}
+
 func appendPodLogToStork(jobName string, namespace string) {
 	// Get job and check whether it has live pod attaced to it
 	job, err := batch.Instance().GetJob(jobName, namespace)
@@ -1752,6 +1807,7 @@ func startTransferJob(
 	drv drivers.Interface,
 	srcPVCName string,
 	compressionType string,
+	excludeFileList string,
 	dataExport *kdmpapi.DataExport,
 	podDataPath string,
 	jobConfigMap string,
@@ -1812,6 +1868,7 @@ func startTransferJob(
 			drivers.WithCertSecretName(utils.GetCertSecretName(dataExport.GetName())),
 			drivers.WithCertSecretNamespace(dataExport.Spec.Source.Namespace),
 			drivers.WithCompressionType(compressionType),
+			drivers.WithExcludeFileList(excludeFileList),
 			drivers.WithPodDatapathType(podDataPath),
 			drivers.WithJobConfigMap(jobConfigMap),
 			drivers.WithJobConfigMapNs(jobConfigMapNs),

pkg/drivers/kopiabackup/kopiabackup.go

@@ -292,6 +292,13 @@ func jobFor(
 		splitCmd = append(splitCmd, "--compression", jobOption.Compression)
 		cmd = strings.Join(splitCmd, " ")
 	}
+
+	if jobOption.ExcludeFileList != "" {
+		splitCmd := strings.Split(cmd, " ")
+		splitCmd = append(splitCmd, "--exclude-file-list", jobOption.ExcludeFileList)
+		cmd = strings.Join(splitCmd, " ")
+	}
+
 	kopiaExecutorImage, imageRegistrySecret, err := utils.GetExecutorImageAndSecret(drivers.KopiaExecutorImage,
 		jobOption.KopiaImageExecutorSource,
 		jobOption.KopiaImageExecutorSourceNs,

pkg/drivers/options.go

@@ -37,6 +37,7 @@ type JobOpts struct {
 	MaintenanceType             string
 	RepoPVCName                 string
 	Compression                 string
+	ExcludeFileList             string
 	PodDataPath                 string
 	// JobConfigMap holds any config needs to be provided to job
 	// from the caller. Eg: executor image name, secret, etc..
@@ -456,6 +457,14 @@ func WithCompressionType(compressionType string) JobOption {
 	}
 }
 
+// WithExcludeFileList is job parameter.
+func WithExcludeFileList(excludeFileList string) JobOption {
+	return func(opts *JobOpts) error {
+		opts.ExcludeFileList = excludeFileList
+		return nil
+	}
+}
+
 // WithPodDatapathType is job parameter.
 func WithPodDatapathType(podDataPath string) JobOption {
 	return func(opts *JobOpts) error {

pkg/executor/common.go

@@ -37,7 +37,7 @@ const (
 	secretAccessKeyPath   = "/etc/cred-secret/secretAccessKey"
 	bucketPath            = "/etc/cred-secret/path"
 	endpointPath          = "/etc/cred-secret/endpoint"
-	sseTypePath          = "/etc/cred-secret/sse"
+	sseTypePath           = "/etc/cred-secret/sse"
 	passwordPath          = "/etc/cred-secret/password"
 	regionPath            = "/etc/cred-secret/region"
 	disableSslPath        = "/etc/cred-secret/disablessl"
@@ -91,7 +91,7 @@ type S3Config struct {
 	// Region will be defaulted to us-east-1 if not provided
 	Region     string
 	DisableSSL bool
-	SseType string
+	SseType    string
 }
 
 // AzureConfig specifies the config required to connect to Azure Blob Storage

pkg/executor/kopia/kopiabackup.go

@@ -32,8 +32,9 @@ const (
 )
 
 var (
-	bkpNamespace string
-	compression  string
+	bkpNamespace    string
+	compression     string
+	excludeFileList string
 )
 
 var (
@@ -67,6 +68,7 @@ func newBackupCommand() *cobra.Command {
 	backupCommand.Flags().StringVar(&sourcePath, "source-path", "", "Source for kopia backup")
 	backupCommand.Flags().StringVar(&sourcePathGlob, "source-path-glob", "", "The regexp should match only one path that will be used for backup")
 	backupCommand.Flags().StringVar(&compression, "compression", "", "Compression type to be used")
+	backupCommand.Flags().StringVar(&excludeFileList, "exclude-file-list", "", " list of dir names that need to be exclude in the kopia snapshot")
 
 	return backupCommand
 }
@@ -164,6 +166,15 @@ func runBackup(sourcePath string) error {
 		}
 	}
 
+	// if excludeFileList is not set in config map, it means no need to exclude any dir in the snapshot.
+	if excludeFileList != "" {
+		if err = runKopiaExcludeFileList(repo, sourcePath); err != nil {
+			errMsg := fmt.Sprintf("setting exclude file list failed for path %s: %v", sourcePath, err)
+			logrus.Errorf("%s: %v", fn, errMsg)
+			return fmt.Errorf(errMsg)
+		}
+	}
+
 	if err = runKopiaBackup(repo, sourcePath); err != nil {
 		errMsg := fmt.Sprintf("backup failed for repository %s: %v", repo.Name, err)
 		logrus.Errorf("%s: %v", fn, errMsg)
@@ -463,6 +474,52 @@ func setGlobalPolicy() error {
 	return nil
 }
 
+func runKopiaExcludeFileList(repository *executor.Repository, sourcePath string) error {
+	logrus.Infof("setting exclude file list for the snapshot")
+	excludeFileListCmd, err := kopia.GetExcludeFileListCommand(
+		sourcePath,
+		excludeFileList,
+	)
+	if err != nil {
+		return err
+	}
+	excludeFileListExecutor := kopia.NewExcludeFileListExecutor(excludeFileListCmd)
+	if err := excludeFileListExecutor.Run(); err != nil {
+		err = fmt.Errorf("failed to run exclude file list command: %v", err)
+		return err
+	}
+	t := func() (interface{}, bool, error) {
+		status, err := excludeFileListExecutor.Status()
+		if err != nil {
+			return "", true, err
+		}
+		if status.LastKnownError != nil {
+			if err = executor.WriteVolumeBackupStatus(
+				status,
+				volumeBackupName,
+				bkpNamespace,
+			); err != nil {
+				errMsg := fmt.Sprintf("failed to write a VolumeBackup status: %v", err)
+				logrus.Errorf("%v", errMsg)
+				return "", true, fmt.Errorf(errMsg)
+			}
+			return "", true, status.LastKnownError
+		}
+		if status.Done {
+			return "", false, nil
+		}
+
+		return "", true, fmt.Errorf("setting exclude file list for snapshot command status not available")
+	}
+	if _, err := task.DoRetryWithTimeout(t, executor.DefaultTimeout, progressCheckInterval); err != nil {
+		logrus.Errorf("failed setting snapshot exclude file list for path %v: %v", sourcePath, err)
+		return err
+	}
+
+	logrus.Infof("setting exclude file list is  successfully")
+	return nil
+}
+
 func runKopiaCompression(repository *executor.Repository, sourcePath string) error {
 	logrus.Infof("Compression started")
 	compressionCmd, err := kopia.GetCompressionCommand(

pkg/kopia/command.go

@@ -3,6 +3,7 @@ package kopia
 import (
 	"os"
 	"os/exec"
+	"strings"
 
 	cmdexec "github.com/portworx/kdmp/pkg/executor"
 	"github.com/sirupsen/logrus"
@@ -46,6 +47,8 @@ type Command struct {
 	DisableSsl bool
 	// Compression to be used for backup
 	Compression string
+	// ExcludeFileList to be used for backup
+	ExcludeFileList string
 	// Region for S3 object
 	Region string
 }
@@ -487,3 +490,32 @@ func (c *Command) CompressionCmd() *exec.Cmd {
 
 	return cmd
 }
+
+// ExcludeFileListCmd returns os/exec.Cmd object for the kopia policy set
+func (c *Command) ExcludeFileListCmd() *exec.Cmd {
+	// Get all the flags
+	argsSlice := []string{
+		c.Name, // compression command
+		"set",
+		c.Path,
+		"--log-dir",
+		logDir,
+		"--config-file",
+		configFile,
+	}
+	commaSplit := strings.Split(c.ExcludeFileList, ",")
+	for _, file := range commaSplit {
+		argsSlice = append(argsSlice, "--add-ignore")
+		argsSlice = append(argsSlice, file)
+	}
+	argsSlice = append(argsSlice, c.Flags...)
+	// Get the cmd args
+	argsSlice = append(argsSlice, c.Args...)
+	cmd := exec.Command(baseCmd, argsSlice...)
+	if len(c.Env) > 0 {
+		cmd.Env = append(os.Environ(), c.Env...)
+	}
+	cmd.Dir = c.Dir
+	logrus.Infof("ExcludeFileListCmd: %+v", cmd)
+	return cmd
+}