Skip to content

Commit

Permalink
Refactoring history-clean command.
Browse files Browse the repository at this point in the history 
Remove --deleted flag.
Only deleted backups are deleted. In order to delete information about backup from the database, it must have the deleted status.
  • Loading branch information
@woblerr
woblerr committed Sep 27, 2024
1 parent 958e6d8 commit 5e36738
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 83 deletions.
39 changes: 10 additions & 29 deletions cmd/history-clean.go → cmd/history_clean.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
var (
historyCleanBeforeTimestamp string
historyCleanOlderThenDays uint
historyCleanDeleted bool
)

var historyCleanCmd = &cobra.Command{
Expand All @@ -23,9 +22,7 @@ var historyCleanCmd = &cobra.Command{
Long: `Clean failed and deleted backups from the history database.
Only the database is being cleaned up.
By default, information is deleted only about failed backups from gpbackup_history.db.
To delete information about deleted backups, use the --deleted option.
Information is deleted only about deleted backups from gpbackup_history.db. Each backup must be deleted first.
To delete information about backups older than the given timestamp, use the --before-timestamp option.
To delete information about backups older than the given number of days, use the --older-than-day option.
Expand Down Expand Up @@ -63,12 +60,6 @@ func init() {
"",
"delete information about backups older than the given timestamp",
)
historyCleanCmd.Flags().BoolVar(
&historyCleanDeleted,
deletedFlagName,
false,
"delete information about deleted backups",
)
historyCleanCmd.MarkFlagsMutuallyExclusive(beforeTimestampFlagName, olderThenDaysFlagName)
}

Expand Down Expand Up @@ -114,7 +105,7 @@ func cleanHistory() error {
gplog.Error(textmsg.ErrorTextUnableActionHistoryDB("close", closeErr))
}
}()
err = historyCleanDB(beforeTimestamp, historyCleanDeleted, hDB)
err = historyCleanDB(beforeTimestamp, hDB)
if err != nil {
return err
}
Expand All @@ -126,7 +117,7 @@ func cleanHistory() error {
return err
}
if len(parseHData.BackupConfigs) != 0 {
err = historyCleanFile(beforeTimestamp, historyCleanDeleted, &parseHData)
err = historyCleanFile(beforeTimestamp, &parseHData)
if err != nil {
return err
}
Expand All @@ -141,15 +132,15 @@ func cleanHistory() error {
return nil
}

func historyCleanDB(cutOffTimestamp string, cleanDeleted bool, hDB *sql.DB) error {
backupList, err := gpbckpconfig.GetBackupNamesForCleanBeforeTimestamp(cutOffTimestamp, cleanDeleted, hDB)
func historyCleanDB(cutOffTimestamp string, hDB *sql.DB) error {
backupList, err := gpbckpconfig.GetBackupNamesForCleanBeforeTimestamp(cutOffTimestamp, hDB)
if err != nil {
gplog.Error(textmsg.ErrorTextUnableReadHistoryDB(err))
return err
}
if len(backupList) > 0 {
gplog.Debug(textmsg.InfoTextBackupDeleteListFromHistory(backupList))
err := gpbckpconfig.CleanBackupsDB(backupList, sqliteDeleteBatchSize, cleanDeleted, hDB)
err := gpbckpconfig.CleanBackupsDB(backupList, sqliteDeleteBatchSize, hDB)
if err != nil {
return err
}
Expand All @@ -159,30 +150,20 @@ func historyCleanDB(cutOffTimestamp string, cleanDeleted bool, hDB *sql.DB) erro
return nil
}

func historyCleanFile(cutOffTimestamp string, cleanDeleted bool, parseHData *gpbckpconfig.History) error {
func historyCleanFile(cutOffTimestamp string, parseHData *gpbckpconfig.History) error {
backupIdxs := make([]int, 0)
backupList := make([]string, 0)
for idx, backupConfig := range parseHData.BackupConfigs {
// In history file we have sorted timestamps by descending order.
if backupConfig.Timestamp < cutOffTimestamp {
backupSuccessStatus, err := backupConfig.IsSuccess()
backupDateDeleted, err := backupConfig.GetBackupDateDeleted()
if err != nil {
gplog.Error(textmsg.ErrorTextUnableGetBackupValue("status", backupConfig.Timestamp, err))
gplog.Error(textmsg.ErrorTextUnableGetBackupValue("date deletion", backupConfig.Timestamp, err))
return err
}
if !backupSuccessStatus {
if !gpbckpconfig.IsBackupActive(backupDateDeleted) && (backupDateDeleted != gpbckpconfig.DateDeletedInProgress) {
backupIdxs = append(backupIdxs, idx)
backupList = append(backupList, backupConfig.Timestamp)
} else if cleanDeleted {
backupDateDeleted, errDateDeleted := backupConfig.GetBackupDateDeleted()
if errDateDeleted != nil {
gplog.Error(textmsg.ErrorTextUnableGetBackupValue("date deletion", backupConfig.Timestamp, errDateDeleted))
return err
}
if !gpbckpconfig.IsBackupActive(backupDateDeleted) && (backupDateDeleted != gpbckpconfig.DateDeletedInProgress) {
backupIdxs = append(backupIdxs, idx)
backupList = append(backupList, backupConfig.Timestamp)
}
}
}
}
Expand Down
80 changes: 38 additions & 42 deletions gpbckpconfig/utils_db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ func GetBackupNamesBeforeTimestamp(timestamp string, historyDB *sql.DB) ([]strin
return execQueryFunc(getBackupNameBeforeTimestampQuery(timestamp), historyDB)
}

func GetBackupNamesForCleanBeforeTimestamp(timestamp string, cleanD bool, historyDB *sql.DB) ([]string, error) {
return execQueryFunc(getBackupNameForCleanBeforeTimestampQuery(timestamp, cleanD), historyDB)
func GetBackupNamesForCleanBeforeTimestamp(timestamp string, historyDB *sql.DB) ([]string, error) {
return execQueryFunc(getBackupNameForCleanBeforeTimestampQuery(timestamp), historyDB)
}

func getBackupNameQuery(showD, showF bool) string {
Expand Down Expand Up @@ -73,7 +73,7 @@ ORDER BY timestamp DESC;
`, backupName, backupName)
}

// Only active backups, "In progress", deleted and failed statuses - hidden.
// Only active backups, "In progress", deleted and failed statuses - hidden.
func getBackupNameBeforeTimestampQuery(timestamp string) string {
return fmt.Sprintf(`
SELECT timestamp
Expand All @@ -82,21 +82,19 @@ WHERE timestamp < '%s'
AND status != '%s'
AND date_deleted IN ('', '%s', '%s')
ORDER BY timestamp DESC;
`, timestamp, BackupStatusFailure, DateDeletedPluginFailed, DateDeletedLocalFailed)
`, timestamp, BackupStatusInProgress, DateDeletedPluginFailed, DateDeletedLocalFailed)
}

func getBackupNameForCleanBeforeTimestampQuery(timestamp string, cleanD bool) string {
orderBy := "ORDER BY timestamp DESC;"
getBackupsQuery := fmt.Sprintf("SELECT timestamp FROM backups WHERE timestamp < '%s'", timestamp)
switch {
case cleanD:
// Return deleted, failed backup.
getBackupsQuery = fmt.Sprintf("%s AND (status = '%s' OR date_deleted NOT IN ('', '%s', '%s', '%s')) %s", getBackupsQuery, BackupStatusFailure, DateDeletedPluginFailed, DateDeletedLocalFailed, DateDeletedInProgress, orderBy)
default:
// Return failed backups.
getBackupsQuery = fmt.Sprintf("%s AND status = '%s' %s", getBackupsQuery, BackupStatusFailure, orderBy)
}
return getBackupsQuery
// Only deleted backups.
func getBackupNameForCleanBeforeTimestampQuery(timestamp string) string {
return fmt.Sprintf(`
SELECT timestamp
FROM backups
WHERE timestamp < '%s'
AND date_deleted NOT IN ('', '%s', '%s', '%s')
ORDER BY timestamp DESC;
`, timestamp, DateDeletedPluginFailed, DateDeletedLocalFailed, DateDeletedInProgress)

}

// UpdateDeleteStatus Updates the date_deleted column in the history database.
Expand All @@ -109,7 +107,7 @@ func UpdateDeleteStatus(backupName, dateDeleted string, historyDB *sql.DB) error
}

// CleanBackupsDB cleans the backup history database by deleting backups based on the given list of backup names.
func CleanBackupsDB(list []string, batchSize int, cleanD bool, historyDB *sql.DB) error {
func CleanBackupsDB(list []string, batchSize int, historyDB *sql.DB) error {
for i := 0; i < len(list); i += batchSize {
end := i + batchSize
if end > len(list) {
Expand All @@ -121,31 +119,29 @@ func CleanBackupsDB(list []string, batchSize int, cleanD bool, historyDB *sql.DB
if err != nil {
return err
}
if cleanD {
err = execStatementFunc(deleteBackupsFormTableQuery("restore_plans", idStr), historyDB)
if err != nil {
return err
}
err = execStatementFunc(deleteBackupsFormTableQuery("restore_plan_tables", idStr), historyDB)
if err != nil {
return err
}
err = execStatementFunc(deleteBackupsFormTableQuery("exclude_relations", idStr), historyDB)
if err != nil {
return err
}
err = execStatementFunc(deleteBackupsFormTableQuery("exclude_schemas", idStr), historyDB)
if err != nil {
return err
}
err = execStatementFunc(deleteBackupsFormTableQuery("include_relations", idStr), historyDB)
if err != nil {
return err
}
err = execStatementFunc(deleteBackupsFormTableQuery("include_schemas", idStr), historyDB)
if err != nil {
return err
}
err = execStatementFunc(deleteBackupsFormTableQuery("restore_plans", idStr), historyDB)
if err != nil {
return err
}
err = execStatementFunc(deleteBackupsFormTableQuery("restore_plan_tables", idStr), historyDB)
if err != nil {
return err
}
err = execStatementFunc(deleteBackupsFormTableQuery("exclude_relations", idStr), historyDB)
if err != nil {
return err
}
err = execStatementFunc(deleteBackupsFormTableQuery("exclude_schemas", idStr), historyDB)
if err != nil {
return err
}
err = execStatementFunc(deleteBackupsFormTableQuery("include_relations", idStr), historyDB)
if err != nil {
return err
}
err = execStatementFunc(deleteBackupsFormTableQuery("include_schemas", idStr), historyDB)
if err != nil {
return err
}
}
return nil
Expand Down
22 changes: 10 additions & 12 deletions gpbckpconfig/utils_db_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ ORDER BY timestamp DESC;
SELECT timestamp
FROM backups
WHERE timestamp < '20240101120000'
AND status != 'Failure'
AND status != 'In Progress'
AND date_deleted IN ('', 'Plugin Backup Delete Failed', 'Local Delete Failed')
ORDER BY timestamp DESC;
`},
Expand All @@ -91,22 +91,20 @@ func TestGetBackupNameForCleanBeforeTimestampQuery(t *testing.T) {
want string
}{
{
name: "Show deleted and failed backups",
name: "Show backups",
value: "20240101120000",
showD: true,
want: `SELECT timestamp FROM backups WHERE timestamp < '20240101120000' AND (status = 'Failure' OR date_deleted NOT IN ('', 'Plugin Backup Delete Failed', 'Local Delete Failed', 'In progress')) ORDER BY timestamp DESC;`,
},
{
name: "Show only failed backups",
value: "20240101120000",
showD: false,
want: `SELECT timestamp FROM backups WHERE timestamp < '20240101120000' AND status = 'Failure' ORDER BY timestamp DESC;`,
},
want: `
SELECT timestamp
FROM backups
WHERE timestamp < '20240101120000'
AND date_deleted NOT IN ('', 'Plugin Backup Delete Failed', 'Local Delete Failed', 'In progress')
ORDER BY timestamp DESC;
`},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := getBackupNameForCleanBeforeTimestampQuery(tt.value, tt.showD); got != tt.want {
if got := getBackupNameForCleanBeforeTimestampQuery(tt.value); got != tt.want {
t.Errorf("getBackupNameForCleanBeforeTimestampQuery(%v, %v):\n%v\nwant:\n%v", tt.value, tt.showD, got, tt.want)
}
})
Expand Down

0 comments on commit 5e36738

Please sign in to comment.