diff --git a/COMMANDS.md b/COMMANDS.md index 2a2057c..bcb92cd 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -40,11 +40,25 @@ Only --older-than-days or --before-timestamp option must be specified, not both. By default, the existence of dependent backups is checked and deletion process is not performed, unless the --cascade option is passed in. -By default, the deletion will be performed for local backup (in development). +By default, the deletion will be performed for local backup. + +The full path to the backup directory can be set using the --backup-dir option. + +For local backups the following logic are applied: + * If the --backup-dir option is specified, the deletion will be performed in provided path. + * If the --backup-dir option is not specified, but the backup was made with --backup-dir flag for gpbackup, the deletion will be performed in the backup manifest path. + * If the --backup-dir option is not specified and backup directory is not specified in backup manifest, the deletion will be performed in backup folder in the master and segments data directories. + * If backup is not local, the error will be returned. + +For control over the number of parallel processes and ssh connections to delete local backups, the --parallel-processes option can be used. The storage plugin config file location can be set using the --plugin-config option. The full path to the file is required. In this case, the deletion will be performed using the storage plugin. +For non local backups the following logic are applied: + * If the --plugin-config option is specified, the deletion will be performed using the storage plugin. + * If backup is local, the error will be returned. + The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. @@ -59,10 +73,12 @@ Usage: gpbackman backup-clean [flags] Flags: + --backup-dir string the full path to backup directory for local backups --before-timestamp string delete backup sets older than the given timestamp --cascade delete all dependent backups -h, --help help for backup-clean --older-than-days uint delete backup sets older than the given number of days + --parallel-processes int the number of parallel processes to delete local backups (default 1) --plugin-config string the full path to plugin config file Global Flags: @@ -76,11 +92,18 @@ Global Flags: ## Examples ### Delete all backups from local storage older than the specified time condition -The functionality is in development. +Delete specific backup with specifying directory path: +```bash +./gpbackman backup-clean \ + --before-timestamp 20240701100000 \ + --cascade +``` -gpBackMan returns a message: +Delete specific backup with specifying the number of parallel processes: ```bash -[WARNING]:-The functionality is still in development +./gpbackman backup-delete \ + --older-than-days 7 \ + --parallel-processes 5 ``` ### Delete all backups using storage plugin older than n days @@ -169,7 +192,7 @@ Usage: gpbackman backup-delete [flags] Flags: - --backup-dir string the full path to backup directory + --backup-dir string the full path to backup directory for local backups --cascade delete all dependent backups for the specified backup timestamp --force try to delete, even if the backup already mark as deleted -h, --help help for backup-delete @@ -199,7 +222,7 @@ Delete specific backup with specifying the number of parallel processes: ```bash ./gpbackman backup-delete \ --timestamp 20230809212220 \ - --parallel-processes 5 + --parallel-processes 5 ``` ### Delete existing backup using storage plugin diff --git a/README.md b/README.md index bf22931..1d7a7b1 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,6 @@ [![Coverage Status](https://coveralls.io/repos/github/woblerr/gpbackman/badge.svg?branch=master)](https://coveralls.io/github/woblerr/gpbackman?branch=master) [![Go Report Card](https://goreportcard.com/badge/github.com/woblerr/gpbackman)](https://goreportcard.com/report/github.com/woblerr/gpbackman) - **gpBackMan** is designed to manage backups created by [gpbackup](https://github.com/greenplum-db/gpbackup) on [Greenplum clusters](https://greenplum.org/). The utility works with both history database formats: `gpbackup_history.yaml` file format (before gpbackup `1.29.0`) and `gpbackup_history.db` SQLite format (starting from gpbackup `1.29.0`). diff --git a/cmd/backup_clean.go b/cmd/backup_clean.go index b42068e..78b1898 100644 --- a/cmd/backup_clean.go +++ b/cmd/backup_clean.go @@ -2,6 +2,7 @@ package cmd import ( "database/sql" + "strconv" "github.com/greenplum-db/gp-common-go-libs/gplog" "github.com/greenplum-db/gpbackup/utils" @@ -13,10 +14,12 @@ import ( // Flags for the gpbackman backup-clean command (backupCleanCmd) var ( - backupCleanBeforeTimestamp string - backupCleanPluginConfigFile string - backupCleanOlderThenDays uint - backupCleanCascade bool + backupCleanBeforeTimestamp string + backupCleanPluginConfigFile string + backupCleanBackupDir string + backupCleanOlderThenDays uint + backupCleanParallelProcesses int + backupCleanCascade bool ) var backupCleanCmd = &cobra.Command{ @@ -31,11 +34,25 @@ Only --older-than-days or --before-timestamp option must be specified, not both. By default, the existence of dependent backups is checked and deletion process is not performed, unless the --cascade option is passed in. -By default, the deletion will be performed for local backup (in development). +By default, the deletion will be performed for local backup. + +The full path to the backup directory can be set using the --backup-dir option. + +For local backups the following logic are applied: + * If the --backup-dir option is specified, the deletion will be performed in provided path. + * If the --backup-dir option is not specified, but the backup was made with --backup-dir flag for gpbackup, the deletion will be performed in the backup manifest path. + * If the --backup-dir option is not specified and backup directory is not specified in backup manifest, the deletion will be performed in backup folder in the master and segments data directories. + * If backup is not local, the error will be returned. + +For control over the number of parallel processes and ssh connections to delete local backups, the --parallel-processes option can be used. The storage plugin config file location can be set using the --plugin-config option. The full path to the file is required. In this case, the deletion will be performed using the storage plugin. +For non local backups the following logic are applied: + * If the --plugin-config option is specified, the deletion will be performed using the storage plugin. + * If backup is local, the error will be returned. + The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. @@ -80,6 +97,18 @@ func init() { "", "delete backup sets older than the given timestamp", ) + backupCleanCmd.PersistentFlags().StringVar( + &backupCleanBackupDir, + backupDirFlagName, + "", + "the full path to backup directory for local backups", + ) + backupCleanCmd.PersistentFlags().IntVar( + &backupCleanParallelProcesses, + parallelProcessesFlagName, + 1, + "the number of parallel processes to delete local backups", + ) backupCleanCmd.MarkFlagsMutuallyExclusive(beforeTimestampFlagName, olderThenDaysFlagName) } @@ -98,6 +127,31 @@ func doCleanBackupFlagValidation(flags *pflag.FlagSet) { if flags.Changed(olderThenDaysFlagName) { beforeTimestamp = gpbckpconfig.GetTimestampOlderThen(backupCleanOlderThenDays) } + // backup-dir anf plugin-config flags cannot be used together. + err = checkCompatibleFlags(flags, backupDirFlagName, pluginConfigFileFlagName) + if err != nil { + gplog.Error(textmsg.ErrorTextUnableCompatibleFlags(err, backupDirFlagName, pluginConfigFileFlagName)) + execOSExit(exitErrorCode) + } + // If parallel-processes flag is specified and have correct values. + if flags.Changed(parallelProcessesFlagName) && !gpbckpconfig.IsPositiveValue(backupCleanParallelProcesses) { + gplog.Error(textmsg.ErrorTextUnableValidateFlag(strconv.Itoa(backupCleanParallelProcesses), parallelProcessesFlagName, err)) + execOSExit(exitErrorCode) + } + // plugin-config and parallel-precesses flags cannot be used together. + err = checkCompatibleFlags(flags, parallelProcessesFlagName, pluginConfigFileFlagName) + if err != nil { + gplog.Error(textmsg.ErrorTextUnableCompatibleFlags(err, parallelProcessesFlagName, pluginConfigFileFlagName)) + execOSExit(exitErrorCode) + } + // If backup-dir flag is specified and it exists and the full path is specified. + if flags.Changed(backupDirFlagName) { + err = gpbckpconfig.CheckFullPath(backupCleanBackupDir, checkFileExistsConst) + if err != nil { + gplog.Error(textmsg.ErrorTextUnableValidateFlag(backupCleanBackupDir, backupDirFlagName, err)) + execOSExit(exitErrorCode) + } + } // If plugin-config flag is specified and it exists and the full path is specified. if flags.Changed(pluginConfigFileFlagName) { err = gpbckpconfig.CheckFullPath(backupCleanPluginConfigFile, checkFileExistsConst) @@ -144,7 +198,7 @@ func cleanBackup() error { return err } } else { - err := backupCleanDBLocal() + err := backupCleanDBLocal(backupCleanCascade, beforeTimestamp, backupCleanBackupDir, backupCleanParallelProcesses, hDB) if err != nil { return err } @@ -166,7 +220,7 @@ func cleanBackup() error { err = backupCleanFilePlugin(backupCleanCascade, beforeTimestamp, backupCleanPluginConfigFile, pluginConfig, &parseHData) if err != nil { // In current implementation, there are cases where some backups were deleted, and some were not. - // Foe example, the clean command was executed without --cascade option. + // For example, the clean command was executed without --cascade option. // In this case - metadata backup was deleted, but full + incremental - weren't. // We should update the history file even it error occurred. errUpdateHFile := parseHData.UpdateHistoryFile(hFile) @@ -177,8 +231,12 @@ func cleanBackup() error { return err } } else { - err := backupCleanFileLocal() + err := backupCleanFileLocal(backupCleanCascade, beforeTimestamp, backupCleanBackupDir, backupCleanParallelProcesses, &parseHData) if err != nil { + errUpdateHFile := parseHData.UpdateHistoryFile(hFile) + if errUpdateHFile != nil { + gplog.Error(textmsg.ErrorTextUnableActionHistoryFile("update", errUpdateHFile)) + } return err } } @@ -214,41 +272,63 @@ func backupCleanDBPlugin(deleteCascade bool, cutOffTimestamp, pluginConfigPath s return nil } -func backupCleanFilePlugin(deleteCascade bool, cutOffTimestamp, pluginConfigPath string, pluginConfig *utils.PluginConfig, parseHData *gpbckpconfig.History) error { - backupList := getBackupNamesBeforeTimestampFile(cutOffTimestamp, true, parseHData) - gplog.Debug(textmsg.InfoTextBackupDeleteList(backupList)) - // Execute deletion for each backup. - // Use backupDeleteFilePlugin function from backup-delete command. - // Don't use force deletes and ignore errors for mass deletion. - err := backupDeleteFilePlugin(backupList, deleteCascade, false, false, pluginConfigPath, pluginConfig, parseHData) +func backupCleanDBLocal(deleteCascade bool, cutOffTimestamp, backupDir string, maxParallelProcesses int, hDB *sql.DB) error { + backupList, err := gpbckpconfig.GetBackupNamesBeforeTimestamp(cutOffTimestamp, hDB) if err != nil { + gplog.Error(textmsg.ErrorTextUnableReadHistoryDB(err)) return err } + if len(backupList) > 0 { + gplog.Debug(textmsg.InfoTextBackupDeleteList(backupList)) + err = backupDeleteDBLocal(backupList, backupDir, deleteCascade, false, false, maxParallelProcesses, hDB) + if err != nil { + return err + } + } else { + gplog.Info(textmsg.InfoTextNothingToDo()) + } return nil } -// TODO -func backupCleanDBLocal() error { - gplog.Warn("The functionality is still in development") +func backupCleanFilePlugin(deleteCascade bool, cutOffTimestamp, pluginConfigPath string, pluginConfig *utils.PluginConfig, parseHData *gpbckpconfig.History) error { + backupList := getBackupNamesBeforeTimestampFile(cutOffTimestamp, true, parseHData) + if len(backupList) > 0 { + gplog.Debug(textmsg.InfoTextBackupDeleteList(backupList)) + // Execute deletion for each backup. + // Use backupDeleteFilePlugin function from backup-delete command. + // Don't use force deletes and ignore errors for mass deletion. + err := backupDeleteFilePlugin(backupList, deleteCascade, false, false, pluginConfigPath, pluginConfig, parseHData) + if err != nil { + return err + } + } else { + gplog.Info(textmsg.InfoTextNothingToDo()) + } return nil } -// TODO -func backupCleanFileLocal() error { - gplog.Warn("The functionality is still in development") +func backupCleanFileLocal(deleteCascade bool, cutOffTimestamp, backupDir string, maxParallelProcesses int, parseHData *gpbckpconfig.History) error { + backupList := getBackupNamesBeforeTimestampFile(cutOffTimestamp, false, parseHData) + if len(backupList) > 0 { + gplog.Debug(textmsg.InfoTextBackupDeleteList(backupList)) + err := backupDeleteFileLocal(backupList, backupDir, deleteCascade, false, false, maxParallelProcesses, parseHData) + if err != nil { + return err + } + } else { + gplog.Info(textmsg.InfoTextNothingToDo()) + } return nil } func getBackupNamesBeforeTimestampFile(timestamp string, skipLocalBackup bool, parseHData *gpbckpconfig.History) []string { backupNames := make([]string, 0) - for idx, backupConfig := range parseHData.BackupConfigs { + for _, backupConfig := range parseHData.BackupConfigs { // In history file we have sorted timestamps by descending order. if backupConfig.Timestamp < timestamp { - for i := idx; i < len(parseHData.BackupConfigs); i++ { - backupCanBeDeleted, _ := checkBackupCanBeUsed(false, skipLocalBackup, parseHData.BackupConfigs[i]) - if backupCanBeDeleted { - backupNames = append(backupNames, parseHData.BackupConfigs[i].Timestamp) - } + backupCanBeDeleted, _ := checkBackupCanBeUsed(false, skipLocalBackup, backupConfig) + if backupCanBeDeleted { + backupNames = append(backupNames, backupConfig.Timestamp) } } } diff --git a/cmd/backup_delete.go b/cmd/backup_delete.go index 9183478..d02aefb 100644 --- a/cmd/backup_delete.go +++ b/cmd/backup_delete.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "os/exec" + "strconv" "sync" "time" @@ -113,7 +114,7 @@ func init() { &backupDeleteBackupDir, backupDirFlagName, "", - "the full path to backup directory", + "the full path to backup directory for local backups", ) backupDeleteCmd.PersistentFlags().IntVar( &backupDeleteParallelProcesses, @@ -149,6 +150,11 @@ func doDeleteBackupFlagValidation(flags *pflag.FlagSet) { gplog.Error(textmsg.ErrorTextUnableCompatibleFlags(err, backupDirFlagName, pluginConfigFileFlagName)) execOSExit(exitErrorCode) } + // If parallel-processes flag is specified and have correct values. + if flags.Changed(parallelProcessesFlagName) && !gpbckpconfig.IsPositiveValue(backupDeleteParallelProcesses) { + gplog.Error(textmsg.ErrorTextUnableValidateFlag(strconv.Itoa(backupDeleteParallelProcesses), parallelProcessesFlagName, err)) + execOSExit(exitErrorCode) + } // plugin-config and parallel-precesses flags cannot be used together. err = checkCompatibleFlags(flags, parallelProcessesFlagName, pluginConfigFileFlagName) if err != nil { diff --git a/cmd/wrappers.go b/cmd/wrappers.go index f8cfd0a..07c9aed 100644 --- a/cmd/wrappers.go +++ b/cmd/wrappers.go @@ -144,7 +144,6 @@ func checkBackupCanBeUsed(deleteForce, skipLocalBackup bool, backupData gpbckpco } if !backupSuccessStatus { gplog.Warn(textmsg.InfoTextBackupFailedStatus(backupData.Timestamp)) - gplog.Info(textmsg.InfoTextNothingToDo()) return result, nil } err = checkLocalBackupStatus(skipLocalBackup, backupData.IsLocal()) @@ -168,7 +167,6 @@ func checkBackupCanBeUsed(deleteForce, skipLocalBackup bool, backupData gpbckpco gplog.Error(textmsg.ErrorTextBackupDeleteInProgress(backupData.Timestamp, textmsg.ErrorBackupDeleteInProgressError())) } else { gplog.Debug(textmsg.InfoTextBackupAlreadyDeleted(backupData.Timestamp)) - gplog.Debug(textmsg.InfoTextNothingToDo()) } } // If flag --force is set. diff --git a/gpbckpconfig/utils.go b/gpbckpconfig/utils.go index 2d5c50f..5e776c9 100644 --- a/gpbckpconfig/utils.go +++ b/gpbckpconfig/utils.go @@ -59,6 +59,11 @@ func IsBackupActive(dateDeleted string) bool { dateDeleted == DateDeletedLocalFailed) } +// IsPositiveValue Returns true if the value is positive. +func IsPositiveValue(value int) bool { + return value > 0 +} + // backupPluginCustomReportPath Returns custom report path: // // /gpbackup__report diff --git a/gpbckpconfig/utils_test.go b/gpbckpconfig/utils_test.go index 865e5d4..c9b2813 100644 --- a/gpbckpconfig/utils_test.go +++ b/gpbckpconfig/utils_test.go @@ -132,6 +132,37 @@ func TestIsBackupActive(t *testing.T) { } } +func TestIsPositiveValue(t *testing.T) { + tests := []struct { + name string + value int + want bool + }{ + { + name: "Test positive value", + value: 10, + want: true, + }, + { + name: "Test zero value", + value: 0, + want: false, + }, + { + name: "Test negative value", + value: -5, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsPositiveValue(tt.value); got != tt.want { + t.Errorf("\nIsPositiveValue() got:\n%v\nwant:\n%v", got, tt.want) + } + }) + } +} + func TestBackupS3PluginReportPath(t *testing.T) { tests := []struct { name string