Add new command to delete selected backup increments safely (#244)

- Adds new command to delete selected backup increments - Implements parser and properties for the new command - Moves increment deletion logic to ManifestManager to make it reusable - Implements new controller to take care of increment deletions - Adds new tests - Updates documentation Resolves #190 {minor} Signed-off-by: Esta Nagy <[email protected]>
nagyesta · May 13, 2024 · 5f22dc4 · 5f22dc4
1 parent dbd54e6
commit 5f22dc4
Show file tree

Hide file tree

Showing 17 changed files with 483 additions and 53 deletions.
diff --git a/.gitignore b/.gitignore
@@ -62,3 +62,5 @@ out/
 
 ### VS Code ###
 */.vscode/
+
+file-barj.log
diff --git a/README.md b/README.md
@@ -47,11 +47,14 @@ File BaRJ comes with the following features
 - Duplicate handling (storing duplicates of the same file only once)
 - Deletes left-over files from the restore directory (if they had been in scope for the backup)
 - Merge previous backup increments
+- Delete selected backup increments
 
 ### Planned features
 
-- Delete selected backup increments
 - UI for convenient configuration
+- Rotate encryption keys
+  - Shallow (only rotate KEK)
+  - Deep (rotate DEKs as well)
 
 ## Modules
 

diff --git a/file-barj-core/README.md b/file-barj-core/README.md
@@ -113,6 +113,20 @@ controller.inspectIncrements(System.out);
 controller.inspectContent(Long.MAX_VALUE, outputFile);
 ```
 
+### Deleting the increments of an archive
+
+```java
+//configuring the deletion job
+final var backupDir = Path.of("/backup/directory");
+final var outputFile = Path.of("/backup/directory");
+final var controller = new IncrementDeletionController(backupDir, "file-prefix", null);
+
+//Delete all backup increments:
+// - starting with the one created at 123456
+// - until (exclusive) the next full backup
+controller.deleteIncrementsUntilNextFullBackupAfter(123456L);
+```
+
 ## Further reading
 
 Please read more about the BaRJ backup jobs [here](https://github.com/nagyesta/file-barj/wiki/Backup-job-configuration-tips).
diff --git a/file-barj-core/src/main/java/com/github/nagyesta/filebarj/core/common/ManifestManager.java b/file-barj-core/src/main/java/com/github/nagyesta/filebarj/core/common/ManifestManager.java
@@ -110,5 +110,17 @@ void validate(
      * @param job the job configuration
      * @return the manifests which can act as previous increments of the provided job
      */
-    SortedMap<Integer, BackupIncrementManifest> loadPreviousManifestsForBackup(BackupJobConfiguration job);
+    SortedMap<Integer, BackupIncrementManifest> loadPreviousManifestsForBackup(
+            @NonNull BackupJobConfiguration job);
+
+    /**
+     * Deletes all files of the backup increment represented by the provided manifest from the
+     * backup directory.
+     *
+     * @param backupDirectory the backup directory
+     * @param manifest        the manifest
+     */
+    void deleteIncrement(
+            @NonNull Path backupDirectory,
+            @NonNull BackupIncrementManifest manifest);
 }
diff --git a/...barj-core/src/main/java/com/github/nagyesta/filebarj/core/common/ManifestManagerImpl.java b/...barj-core/src/main/java/com/github/nagyesta/filebarj/core/common/ManifestManagerImpl.java
@@ -33,6 +33,8 @@
  */
 @Slf4j
 public class ManifestManagerImpl implements ManifestManager {
+    private static final String HISTORY_FOLDER = ".history";
+    private static final String MANIFEST_JSON_GZ = ".manifest.json.gz";
     private final ObjectMapper mapper = new ObjectMapper();
 
     @Override
@@ -76,10 +78,10 @@ public void persist(
     private void doPersist(
             @NotNull final BackupIncrementManifest manifest,
             @NotNull final File backupDestination) {
-        final var backupHistoryDir = new File(backupDestination, ".history");
+        final var backupHistoryDir = new File(backupDestination, HISTORY_FOLDER);
         //noinspection ResultOfMethodCallIgnored
         backupHistoryDir.mkdirs();
-        final var plainManifestFile = new File(backupHistoryDir, manifest.getFileNamePrefix() + ".manifest.json.gz");
+        final var plainManifestFile = new File(backupHistoryDir, manifest.getFileNamePrefix() + MANIFEST_JSON_GZ);
         try (var fileStream = new FileOutputStream(plainManifestFile);
              var bufferedStream = new BufferedOutputStream(fileStream);
              var gzipStream = new GZIPOutputStream(bufferedStream);
@@ -140,15 +142,16 @@ public SortedMap<Long, BackupIncrementManifest> loadAll(
     }
 
     @Override
-    public SortedMap<Integer, BackupIncrementManifest> loadPreviousManifestsForBackup(final BackupJobConfiguration job) {
-        final var historyFolder = job.getDestinationDirectory().resolve(".history");
+    public SortedMap<Integer, BackupIncrementManifest> loadPreviousManifestsForBackup(
+            @NonNull final BackupJobConfiguration job) {
+        final var historyFolder = job.getDestinationDirectory().resolve(HISTORY_FOLDER);
         if (!Files.exists(historyFolder)) {
             return Collections.emptySortedMap();
         }
         try (var pathStream = Files.list(historyFolder)) {
             final var manifestFiles = pathStream
                     .filter(path -> path.getFileName().toString().startsWith(job.getFileNamePrefix()))
-                    .filter(path -> path.getFileName().toString().endsWith(".manifest.json.gz"))
+                    .filter(path -> path.getFileName().toString().endsWith(MANIFEST_JSON_GZ))
                     .sorted(Comparator.comparing(Path::getFileName).reversed())
                     .toList();
             final var manifests = loadManifests(manifestFiles, null, Long.MAX_VALUE);
@@ -202,6 +205,15 @@ public void validate(
         //TODO: implement validation
     }
 
+    @Override
+    public void deleteIncrement(
+            @NonNull final Path backupDirectory,
+            @NonNull final BackupIncrementManifest manifest) {
+        final var fileNamePrefix = manifest.getFileNamePrefix();
+        deleteManifestFromHistoryIfExists(backupDirectory, fileNamePrefix);
+        deleteManifestAndArchiveFilesFromBackupDirectory(backupDirectory, fileNamePrefix);
+    }
+
     @NotNull
     private SortedMap<Integer, BackupIncrementManifest> loadManifests(
             @NotNull final List<Path> manifestFiles,
@@ -342,4 +354,45 @@ private SortedMap<Integer, Map<Integer, String>> mergeEncryptionKeys(
                 .forEach(keys::putAll);
         return keys;
     }
+
+    private void deleteManifestAndArchiveFilesFromBackupDirectory(
+            @NotNull final Path backupDirectory, @NotNull final String fileNamePrefix) {
+        final var patterns = Set.of(
+                "^" + fileNamePrefix + "\\.[0-9]{5}\\.cargo$",
+                "^" + fileNamePrefix + "\\.manifest\\.cargo$",
+                "^" + fileNamePrefix + "\\.index\\.cargo$"
+        );
+        try (var list = Files.list(backupDirectory)) {
+            final var toDelete = new ArrayList<Path>();
+            list.filter(path -> patterns.stream().anyMatch(pattern -> path.getFileName().toString().matches(pattern)))
+                    .forEach(toDelete::add);
+            for (final var path : toDelete) {
+                log.info("Deleting obsolete backup file: {}", path);
+                try {
+                    Files.delete(path);
+                } catch (final IOException e) {
+                    log.warn("Unable to delete file! Will attempt to delete it on exit.", e);
+                    if (Files.exists(path)) {
+                        path.toFile().deleteOnExit();
+                    }
+                }
+            }
+        } catch (final IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void deleteManifestFromHistoryIfExists(
+            @NotNull final Path backupDirectory, @NotNull final String fileNamePrefix) {
+        final var fromHistory = backupDirectory.resolve(HISTORY_FOLDER)
+                .resolve(fileNamePrefix + MANIFEST_JSON_GZ);
+        try {
+            if (Files.exists(fromHistory)) {
+                log.info("Deleting obsolete file from history: {}", fromHistory);
+                Files.delete(fromHistory);
+            }
+        } catch (final IOException e) {
+            log.error("Could not delete manifest file from history folder: {}", fromHistory, e);
+        }
+    }
 }
diff --git a/...e/src/main/java/com/github/nagyesta/filebarj/core/delete/IncrementDeletionController.java b/...e/src/main/java/com/github/nagyesta/filebarj/core/delete/IncrementDeletionController.java
@@ -0,0 +1,67 @@
+package com.github.nagyesta.filebarj.core.delete;
+
+import com.github.nagyesta.filebarj.core.common.ManifestManager;
+import com.github.nagyesta.filebarj.core.common.ManifestManagerImpl;
+import com.github.nagyesta.filebarj.core.model.BackupIncrementManifest;
+import com.github.nagyesta.filebarj.core.model.enums.BackupType;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.jetbrains.annotations.Nullable;
+
+import java.nio.file.Path;
+import java.security.PrivateKey;
+import java.util.Comparator;
+import java.util.SortedMap;
+
+/**
+ * Controller for the backup increment deletion task.
+ */
+@Slf4j
+public class IncrementDeletionController {
+
+    private final SortedMap<Long, BackupIncrementManifest> manifests;
+    private final @NonNull Path backupDirectory;
+    private final ManifestManager manifestManager;
+
+    /**
+     * Creates a new instance and initializes it for the specified job.
+     *
+     * @param backupDirectory the directory where the backup files are located
+     * @param fileNamePrefix  the prefix of the backup file names
+     * @param kek             The key encryption key we want to use to decrypt the files (optional).
+     *                        If null, no decryption will be performed.
+     */
+    public IncrementDeletionController(
+            @NonNull final Path backupDirectory,
+            @NonNull final String fileNamePrefix,
+            @Nullable final PrivateKey kek) {
+        this.manifestManager = new ManifestManagerImpl();
+        this.backupDirectory = backupDirectory;
+        this.manifests = this.manifestManager.loadAll(this.backupDirectory, fileNamePrefix, kek);
+    }
+
+    /**
+     * Deletes the incremental backups which were created after the specified time until the next
+     * full backup.
+     *
+     * @param startingWithEpochSeconds the start time of the first deleted increment
+     */
+    public void deleteIncrementsUntilNextFullBackupAfter(final long startingWithEpochSeconds) {
+        final var incrementsStartingWithThreshold = this.manifests.values().stream()
+                .sorted(Comparator.comparing(BackupIncrementManifest::getStartTimeUtcEpochSeconds))
+                .filter(manifest -> manifest.getStartTimeUtcEpochSeconds() >= startingWithEpochSeconds)
+                .toList();
+        if (incrementsStartingWithThreshold.isEmpty()) {
+            throw new IllegalArgumentException("No backups found after: " + startingWithEpochSeconds);
+        }
+        if (incrementsStartingWithThreshold.get(0).getStartTimeUtcEpochSeconds() != startingWithEpochSeconds) {
+            throw new IllegalArgumentException("Unable to find backup which started at: " + startingWithEpochSeconds);
+        }
+        for (final var current : incrementsStartingWithThreshold) {
+            if (current.getStartTimeUtcEpochSeconds() > startingWithEpochSeconds && current.getBackupType() == BackupType.FULL) {
+                break;
+            }
+            manifestManager.deleteIncrement(backupDirectory, current);
+        }
+    }
+}
diff --git a/file-barj-core/src/main/java/com/github/nagyesta/filebarj/core/merge/MergeController.java b/file-barj-core/src/main/java/com/github/nagyesta/filebarj/core/merge/MergeController.java
@@ -16,7 +16,6 @@
 import org.jetbrains.annotations.Nullable;
 
 import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.PrivateKey;
 import java.util.*;
@@ -85,9 +84,7 @@ public BackupIncrementManifest execute(final boolean deleteObsoleteFiles) {
             if (deleteObsoleteFiles) {
                 log.info("Deleting obsolete files from backup directory: {}", backupDirectory);
                 selectedManifests.values().forEach(manifest -> {
-                    final var fileNamePrefix = manifest.getFileNamePrefix();
-                    deleteManifestFromHistoryIfExists(fileNamePrefix);
-                    deleteManifestAndArchiveFilesFromBackupDirectory(fileNamePrefix);
+                    manifestManager.deleteIncrement(backupDirectory, manifest);
                 });
             }
             return result;
@@ -96,45 +93,6 @@ public BackupIncrementManifest execute(final boolean deleteObsoleteFiles) {
         }
     }
 
-    private void deleteManifestAndArchiveFilesFromBackupDirectory(@NotNull final String fileNamePrefix) {
-        final var patterns = Set.of(
-                "^" + fileNamePrefix + "\\.[0-9]{5}\\.cargo$",
-                "^" + fileNamePrefix + "\\.manifest\\.cargo$",
-                "^" + fileNamePrefix + "\\.index\\.cargo$"
-        );
-        try (var list = Files.list(backupDirectory)) {
-            final var toDelete = new ArrayList<Path>();
-            list.filter(path -> patterns.stream().anyMatch(pattern -> path.getFileName().toString().matches(pattern)))
-                    .forEach(toDelete::add);
-            for (final var path : toDelete) {
-                log.info("Deleting obsolete file: {}", path);
-                try {
-                    Files.delete(path);
-                } catch (final IOException e) {
-                    log.warn("Unable to delete file! Will attempt to delete it on exit.", e);
-                    if (Files.exists(path)) {
-                        path.toFile().deleteOnExit();
-                    }
-                }
-            }
-        } catch (final IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    private void deleteManifestFromHistoryIfExists(@NotNull final String fileNamePrefix) {
-        final var fromHistory = backupDirectory.resolve(".history")
-                .resolve(fileNamePrefix + ".manifest.json.gz");
-        try {
-            if (Files.exists(fromHistory)) {
-                log.info("Deleting obsolete file from history: {}", fromHistory);
-                Files.delete(fromHistory);
-            }
-        } catch (final IOException e) {
-            log.error("Could not delete manifest file from history folder: " + fromHistory, e);
-        }
-    }
-
     @NotNull
     private BackupIncrementManifest mergeBackupContent() {
         final var lastManifest = manifestsToMerge.get(manifestsToMerge.lastKey());

diff --git a/...-core/src/test/java/com/github/nagyesta/filebarj/core/common/ManifestManagerImplTest.java b/...-core/src/test/java/com/github/nagyesta/filebarj/core/common/ManifestManagerImplTest.java
@@ -383,4 +383,42 @@ void testMergeForRestoreShouldThrowExceptionWhenCalledWithNull() {
 
         //then + exception
     }
+
+    @SuppressWarnings("DataFlowIssue")
+    @Test
+    void testDeleteIncrementShouldThrowExceptionWhenCalledWithNullManifest() {
+        //given
+        final var underTest = new ManifestManagerImpl();
+
+        //when
+        Assertions.assertThrows(IllegalArgumentException.class, () -> underTest.deleteIncrement(Path.of("destination"), null));
+
+        //then + exception
+    }
+
+    @SuppressWarnings("DataFlowIssue")
+    @Test
+    void testDeleteIncrementShouldThrowExceptionWhenCalledWithNullDestination() {
+        //given
+        final var underTest = new ManifestManagerImpl();
+
+        //when
+        Assertions.assertThrows(IllegalArgumentException.class,
+                () -> underTest.deleteIncrement(null, mock(BackupIncrementManifest.class)));
+
+        //then + exception
+    }
+
+    @SuppressWarnings("DataFlowIssue")
+    @Test
+    void testLoadPreviousManifestsForBackupShouldThrowExceptionWhenCalledWithNull() {
+        //given
+        final var underTest = new ManifestManagerImpl();
+
+        //when
+        Assertions.assertThrows(IllegalArgumentException.class,
+                () -> underTest.loadPreviousManifestsForBackup(null));
+
+        //then + exception
+    }
 }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -62,3 +62,5 @@ out/

		### VS Code ###
		*/.vscode/

		file-barj.log