Add option to merge incremental backups (#158)

- Implements merge functionality
- Merge can delete obsolete backup files
- Enforces hash verification during merge (including during multi-threaded backups)
- Adds new --merge command to the CLI job
- Updates tests
- Updates documentation

Resolves #154
{minor}

Signed-off-by: Esta Nagy <[email protected]>

README.md

@@ -46,10 +46,11 @@ File BaRJ comes with the following features
 - Inspect content of a backup increment
 - Duplicate handling (storing duplicates of the same file only once)
 - Deletes left-over files from the restore directory (if they had been in scope for the backup)
+- Merge previous backup increments
 
 ### Planned features
 
-- Merge previous backup increments
+- Delete selected backup increments
 - UI for convenient configuration
 
 ## Modules

file-barj-core/README.md

@@ -63,6 +63,20 @@ final var backupController = new BackupController(configuration, false);
 backupController.execute(1);
 ```
 
+### Merging increments
+
+```java
+final var mergeController = new MergeController(
+        Path.of("/tmp/backup"), 
+        "prefix", 
+        null, //optional key encryption key
+        123L, //Backup start epoch seconds for the first file of the range (inclusive)
+        234L  //Backup start epoch seconds for the last file of the range (inclusive)
+);
+
+mergeController.execute(false);
+```
+
 ### Reading an archive
 
 ```java

file-barj-core/src/main/java/com/github/nagyesta/filebarj/core/common/ManifestManager.java

@@ -31,14 +31,27 @@ BackupIncrementManifest generateManifest(
             int nextVersion);
 
     /**
-     * Persists the provided manifest.to the hard drive in two copies (one encrypted that can be
+     * Persists the provided manifest to the hard drive in two copies (one encrypted that can be
      * moved to a safe location and one unencrypted in the history folder to allow incremental
      * backup jobs to function automatically without knowing the private keys).
      *
      * @param manifest The manifest to persist
      */
     void persist(@NonNull BackupIncrementManifest manifest);
 
+    /**
+     * Persists the provided manifest to the hard drive in two copies (one encrypted that can be
+     * moved to a safe location and one unencrypted in the history folder to allow incremental
+     * backup jobs to function automatically without knowing the private keys).
+     * The aforementioned files will be stored relative to the provided backup destination.
+     *
+     * @param manifest          The manifest to persist
+     * @param backupDestination the backup destination
+     */
+    void persist(
+            @NonNull BackupIncrementManifest manifest,
+            @NonNull Path backupDestination);
+
     /**
      * Loads the manifests which belong to the provided backup. Only includes manifests starting
      * with the latest full backup before the provided time stamp.
@@ -61,10 +74,10 @@ SortedMap<Integer, BackupIncrementManifest> load(
      * Loads all manifests which belong to the provided backup. Contains manifests for all
      * increments even if many full backups have been created.
      *
-     * @param destinationDirectory    the directory where the backup files are stored
-     * @param fileNamePrefix          the prefix of the backup files
-     * @param privateKey              the RSA key we want to use to decrypt the manifests (optional).
-     *                                If null, the manifests will not be decrypted.
+     * @param destinationDirectory the directory where the backup files are stored
+     * @param fileNamePrefix       the prefix of the backup files
+     * @param privateKey           the RSA key we want to use to decrypt the manifests (optional).
+     *                             If null, the manifests will not be decrypted.
      * @return the map of loaded manifests keyed by their timestamps
      */
     SortedMap<Long, BackupIncrementManifest> loadAll(

file-barj-core/src/main/java/com/github/nagyesta/filebarj/core/common/ManifestManagerImpl.java

@@ -61,8 +61,21 @@ public BackupIncrementManifest generateManifest(
     @Override
     public void persist(
             @NonNull final BackupIncrementManifest manifest) {
+        final var backupDestination = manifest.getConfiguration().getDestinationDirectory();
+        persist(manifest, backupDestination);
+    }
+
+    @Override
+    public void persist(
+            @NonNull final BackupIncrementManifest manifest,
+            @NonNull final Path backupDestination) {
         validate(manifest, ValidationRules.Persisted.class);
-        final var backupDestination = manifest.getConfiguration().getDestinationDirectory().toFile();
+        doPersist(manifest, backupDestination.toFile());
+    }
+
+    private void doPersist(
+            @NotNull final BackupIncrementManifest manifest,
+            @NotNull final File backupDestination) {
         final var backupHistoryDir = new File(backupDestination, ".history");
         //noinspection ResultOfMethodCallIgnored
         backupHistoryDir.mkdirs();
@@ -237,6 +250,9 @@ private SortedMap<Long, BackupIncrementManifest> loadAllManifests(
                 log.warn("Failed to load manifest file: {}", path, e);
             }
         }
+        if (manifests.isEmpty()) {
+            throw new ArchivalException("No manifests found.");
+        }
         return manifests;
     }
 

file-barj-core/src/main/java/com/github/nagyesta/filebarj/core/inspect/worker/ManifestToSummaryConverter.java

@@ -2,8 +2,11 @@
 
 import com.github.nagyesta.filebarj.core.model.BackupIncrementManifest;
 import com.github.nagyesta.filebarj.core.model.FileMetadata;
+import com.github.nagyesta.filebarj.core.model.enums.BackupType;
+import com.github.nagyesta.filebarj.core.util.LogUtil;
 import com.github.nagyesta.filebarj.io.stream.internal.ChunkingOutputStream;
 import lombok.NonNull;
+import org.jetbrains.annotations.NotNull;
 
 import java.time.Instant;
 import java.util.Optional;
@@ -23,7 +26,7 @@ public String convertToSummaryString(final @NonNull BackupIncrementManifest mani
         final var epochSeconds = manifest.getStartTimeUtcEpochSeconds();
         final var totalSize = manifest.getFiles().values().stream()
                 .mapToLong(FileMetadata::getOriginalSizeBytes).sum() / ChunkingOutputStream.MEBIBYTE;
-        return manifest.getBackupType().name() + " backup: " + manifest.getFileNamePrefix() + "\n"
+        return getFormattedType(manifest) + " backup: " + manifest.getFileNamePrefix() + "\n"
                 + "\tStarted at : " + Instant.ofEpochSecond(epochSeconds) + " (Epoch seconds: " + epochSeconds + ")\n"
                 + "\tContains " + manifest.getFiles().size() + " files (" + totalSize + " MiB)\n"
                 + "\tVersions   : " + manifest.getVersions() + "\n"
@@ -32,4 +35,12 @@ public String convertToSummaryString(final @NonNull BackupIncrementManifest mani
                 + "\tHash alg.  : " + manifest.getConfiguration().getHashAlgorithm().name() + "\n"
                 + "\tCompression: " + manifest.getConfiguration().getCompression().name();
     }
+
+    @NotNull
+    private String getFormattedType(@NotNull final BackupIncrementManifest manifest) {
+        if (manifest.getBackupType() == BackupType.INCREMENTAL) {
+            return manifest.getBackupType().name();
+        }
+        return LogUtil.scary(manifest.getBackupType().name());
+    }
 }