Allow partial restore (#126)

- Adds new option to the --restore command to limit the restore scope to certain backup directories - Changes restore implementation to respect the provided filtering predicate - Adds new tests - Updates documentation Resolves #125 {minor} Signed-off-by: Esta Nagy <[email protected]>
nagyesta · Feb 3, 2024 · 39bb62e · 39bb62e
1 parent f955b29
commit 39bb62e
Show file tree

Hide file tree

Showing 12 changed files with 179 additions and 8 deletions.
diff --git a/README.md b/README.md
@@ -40,6 +40,8 @@ File BaRJ comes with the following features
 - Restore/unpack previous backup
   - Using latest increment
   - Using a selected earlier increment
+  - Restoring all files from the backup
+  - Restoring only some included files (partial restore)
 - Inspect available backup increments
 - Inspect content of a backup increment
 - Duplicate handling (storing duplicates of the same file only once)

diff --git a/file-barj-core/README.md b/file-barj-core/README.md
@@ -74,6 +74,7 @@ final var restoreTask = RestoreTask.builder()
         .dryRun(true)
         .threads(1)
         .deleteFilesNotInBackup(false)
+        .includedPath(Path.of("/source/dir")) //optional path filter
         .build();
 final var pointInTime = 123456L;
 final var restoreController = new RestoreController(Path.of("/tmp/backup"), "test", null, pointInTime);

diff --git a/file-barj-core/src/main/java/com/github/nagyesta/filebarj/core/config/RestoreTask.java b/file-barj-core/src/main/java/com/github/nagyesta/filebarj/core/config/RestoreTask.java
@@ -4,6 +4,10 @@
 import lombok.Data;
 import lombok.NonNull;
 
+import java.nio.file.Path;
+import java.util.Optional;
+import java.util.function.Predicate;
+
 /**
  * Defines the parameters of a restore task.
  */
@@ -29,4 +33,19 @@ public class RestoreTask {
      * are not in the backup increment when true.
      */
     private final boolean deleteFilesNotInBackup;
+    /**
+     * The root path of the backup entries (directory or file) to include during the restore.
+     */
+    private final Path includedPath;
+
+    /**
+     * Returns the path filter for this restore task based on the included path.
+     *
+     * @return the path filter
+     */
+    public Predicate<Path> getPathFilter() {
+        return Optional.ofNullable(includedPath)
+                .map(includedPath -> (Predicate<Path>) path -> path.equals(includedPath) || path.startsWith(includedPath))
+                .orElse(path -> true);
+    }
 }
diff --git a/file-barj-core/src/main/java/com/github/nagyesta/filebarj/core/model/RestoreManifest.java b/file-barj-core/src/main/java/com/github/nagyesta/filebarj/core/model/RestoreManifest.java
@@ -2,16 +2,20 @@
 
 import com.github.nagyesta.filebarj.core.config.BackupJobConfiguration;
 import com.github.nagyesta.filebarj.core.model.enums.Change;
+import com.github.nagyesta.filebarj.core.model.enums.FileType;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NonNull;
 import lombok.experimental.SuperBuilder;
 import org.jetbrains.annotations.NotNull;
 
 import javax.crypto.SecretKey;
+import java.nio.file.Path;
 import java.security.PrivateKey;
 import java.util.*;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 @Data
 @SuperBuilder
@@ -83,14 +87,42 @@ public Map<UUID, FileMetadata> getFilesOfLastManifest() {
     }
 
     /**
-     * The map of all files in the manifest. This is a read-only view of the files map.
+     * The map of all files in the manifest which are matching the provided predicate and their
+     * parent directories. This is a read-only view of the files map.
      *
+     * @param predicate the predicate filtering the paths to be returned
      * @return the map
      */
-    public List<FileMetadata> getExistingContentSourceFilesOfLastManifest() {
+    public Map<UUID, FileMetadata> getFilesOfLastManifestFilteredBy(
+            final Predicate<Path> predicate) {
+        final var filesOfLastManifest = getFilesOfLastManifest();
+        final var allDirectories = filesOfLastManifest.values().stream()
+                .filter(fileMetadata -> fileMetadata.getFileType() == FileType.DIRECTORY)
+                .map(FileMetadata::getAbsolutePath)
+                .collect(Collectors.toSet());
+        final var foundPaths = filesOfLastManifest.values().stream()
+                .map(FileMetadata::getAbsolutePath)
+                .filter(predicate)
+                .flatMap(path -> parents(allDirectories, path))
+                .collect(Collectors.toSet());
+        return filesOfLastManifest.entrySet().stream()
+                .filter(e -> foundPaths.contains(e.getValue().getAbsolutePath()))
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+    }
+
+    /**
+     * The map of all files in the manifest which are matching the provided predicate and their
+     * parent directories.. This is a read-only view of the files map.
+     *
+     * @param predicate the predicate filtering the paths to be returned
+     * @return the list
+     */
+    public List<FileMetadata> getExistingContentSourceFilesOfLastManifestFilteredBy(
+            final Predicate<Path> predicate) {
         return getFilesOfLastManifest().values().stream()
                 .filter(fileMetadata -> fileMetadata.getStatus() != Change.DELETED)
                 .filter(fileMetadata -> fileMetadata.getFileType().isContentSource())
+                .filter(fileMetadata -> predicate.test(fileMetadata.getAbsolutePath()))
                 .toList();
     }
 
@@ -102,4 +134,11 @@ public List<FileMetadata> getExistingContentSourceFilesOfLastManifest() {
     public Map<UUID, ArchivedFileMetadata> getArchivedEntriesOfLastManifest() {
         return archivedEntries.get(fileNamePrefixes.lastKey());
     }
+
+    private Stream<Path> parents(final Set<Path> directories, final Path path) {
+        return Optional.ofNullable(path.getParent())
+                .filter(directories::contains)
+                .map(parent -> Stream.concat(Stream.of(path), parents(directories, parent)))
+                .orElse(Stream.of(path));
+    }
 }
diff --git a/...e/src/main/java/com/github/nagyesta/filebarj/core/restore/pipeline/RestoreController.java b/...e/src/main/java/com/github/nagyesta/filebarj/core/restore/pipeline/RestoreController.java
@@ -4,6 +4,7 @@
 import com.github.nagyesta.filebarj.core.common.ManifestManagerImpl;
 import com.github.nagyesta.filebarj.core.config.RestoreTargets;
 import com.github.nagyesta.filebarj.core.config.RestoreTask;
+import com.github.nagyesta.filebarj.core.inspect.worker.ManifestToSummaryConverter;
 import com.github.nagyesta.filebarj.core.model.FileMetadata;
 import com.github.nagyesta.filebarj.core.model.RestoreManifest;
 import com.github.nagyesta.filebarj.core.model.enums.FileType;
@@ -66,6 +67,8 @@ public RestoreController(
         final ManifestManager manifestManager = new ManifestManagerImpl();
         log.info("Loading backup manifests for restore from: {}", backupDirectory);
         final var manifests = manifestManager.load(backupDirectory, fileNamePrefix, kek, atPointInTime);
+        final var header = new ManifestToSummaryConverter().convertToSummaryString(manifests.get(manifests.lastKey()));
+        log.info("Latest backup manifest: {}", header);
         log.info("Merging {} manifests", manifests.size());
         manifest = manifestManager.mergeForRestore(manifests);
         final var filesOfLastManifest = manifest.getFilesOfLastManifest();
@@ -86,8 +89,8 @@ public void execute(
         executionLock.lock();
         try {
             this.threadPool = new ForkJoinPool(restoreTask.getThreads());
-            final var allEntries = manifest.getFilesOfLastManifest().values().stream().toList();
-            final var contentSources = manifest.getExistingContentSourceFilesOfLastManifest();
+            final var allEntries = manifest.getFilesOfLastManifestFilteredBy(restoreTask.getPathFilter()).values().stream().toList();
+            final var contentSources = manifest.getExistingContentSourceFilesOfLastManifestFilteredBy(restoreTask.getPathFilter());
             final long totalBackupSize = allEntries.stream()
                     .map(FileMetadata::getOriginalSizeBytes)
                     .reduce(0L, Long::sum);
@@ -100,7 +103,7 @@ public void execute(
                     .filter(metadata -> metadata.getFileType() == FileType.DIRECTORY)
                     .toList());
             pipeline.restoreFiles(contentSources, threadPool);
-            pipeline.deleteLeftOverFiles(restoreTask.isDeleteFilesNotInBackup(), threadPool);
+            pipeline.deleteLeftOverFiles(restoreTask.getIncludedPath(), restoreTask.isDeleteFilesNotInBackup(), threadPool);
             pipeline.finalizePermissions(allEntries, threadPool);
             pipeline.evaluateRestoreSuccess(allEntries, threadPool);
             final var endTimeMillis = System.currentTimeMillis();

diff --git a/...ore/src/main/java/com/github/nagyesta/filebarj/core/restore/pipeline/RestorePipeline.java b/...ore/src/main/java/com/github/nagyesta/filebarj/core/restore/pipeline/RestorePipeline.java
@@ -41,6 +41,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 import static com.github.nagyesta.filebarj.core.util.TimerUtil.toProcessSummary;
@@ -163,15 +164,22 @@ public void finalizePermissions(
     /**
      * Deletes the left-over files in the restore target.
      *
+     * @param includedPath        The root path included in the restore task
      * @param deleteLeftOverFiles True if we should delete left-over files
      * @param threadPool          The thread pool we can use for parallel processing
      */
-    public void deleteLeftOverFiles(final boolean deleteLeftOverFiles, @NonNull final ForkJoinPool threadPool) {
+    public void deleteLeftOverFiles(final Path includedPath, final boolean deleteLeftOverFiles, @NonNull final ForkJoinPool threadPool) {
         if (!deleteLeftOverFiles) {
             log.info("Skipping left-over files deletion...");
             return;
         }
-        log.info("Deleting left-over files (if any)...");
+        final var pathRestriction = Optional.ofNullable(includedPath)
+                .map(restoreTargets::mapToRestorePath)
+                .map(included -> (Predicate<Path>) path -> path.equals(included) || path.startsWith(included))
+                .orElse(path -> true);
+        log.info("Deleting left-over files (if any){}", Optional.ofNullable(includedPath)
+                .map(path -> " limited to path: " + path)
+                .orElse(""));
         final var files = manifest.getFilesOfLastManifest().values();
         final var modifiedSources = manifest.getConfiguration().getSources().stream()
                 .map(source -> BackupSource.builder()
@@ -183,6 +191,7 @@ public void deleteLeftOverFiles(final boolean deleteLeftOverFiles, @NonNull fina
         final var pathsInRestoreTarget = modifiedSources.stream()
                 .map(BackupSource::listMatchingFilePaths)
                 .flatMap(Collection::stream)
+                .filter(pathRestriction)
                 .collect(Collectors.toSet());
         final var pathsInBackup = threadPool.submit(() -> files.parallelStream()
                 .map(FileMetadata::getAbsolutePath)

diff --git a/.../com/github/nagyesta/filebarj/core/restore/pipeline/RestoreControllerIntegrationTest.java b/.../com/github/nagyesta/filebarj/core/restore/pipeline/RestoreControllerIntegrationTest.java
@@ -166,6 +166,86 @@ void testExecuteShouldRestoreFilesToDestinationWhenExecutedWithValidInput(
         assertFileMetadataMatches(sourceFolder, realRestorePath.resolve("folder"), metadataParser, configuration);
     }
 
+    @ParameterizedTest
+    @MethodSource("restoreParameterProvider")
+    void testExecuteShouldRestoreOnlyIncludedFilesToDestinationWhenExecutedWithIncludeFilter(
+            final PublicKey encryptionKey, final PrivateKey decryptionKey,
+            final int threads, final HashAlgorithm hash) throws IOException {
+        //given
+        final var sourceDir = testDataRoot.resolve("source-dir" + UUID.randomUUID());
+        final var backupDir = testDataRoot.resolve("backup-dir" + UUID.randomUUID());
+        final var movedBackupDir = testDataRoot.resolve("moved-backup-dir" + UUID.randomUUID());
+        final var restoreDir = testDataRoot.resolve("restore-dir" + UUID.randomUUID());
+        final var configuration = getBackupJobConfiguration(BackupType.FULL, sourceDir, backupDir, encryptionKey, hash);
+
+        final var aPng = sourceDir.resolve("A.png");
+        final var bPng = sourceDir.resolve("B.png");
+        final var cPng = sourceDir.resolve("C.png");
+        final var sourceFiles = List.of(aPng, bPng, cPng);
+        for (final var sourceFile : sourceFiles) {
+            FileUtils.copyFile(getExampleResource(), sourceFile.toFile());
+        }
+        final var sourceFolder = sourceDir.resolve("folder");
+        Files.createDirectories(sourceFolder);
+
+        final var sourceLinkInternal = sourceDir.resolve("folder/internal.png");
+        Files.createSymbolicLink(sourceLinkInternal, aPng);
+        final var sourceLinkExternal = sourceDir.resolve("external.png");
+        final var externalLinkTarget = getExampleResource().toPath().toAbsolutePath();
+        Files.createSymbolicLink(sourceLinkExternal, externalLinkTarget);
+
+        final var backupController = new BackupController(configuration, true);
+        backupController.execute(1);
+
+        Files.move(backupDir, movedBackupDir);
+
+        final var underTest = new RestoreController(
+                movedBackupDir, configuration.getFileNamePrefix(), decryptionKey);
+        final var restoreTargets = new RestoreTargets(Set.of(new RestoreTarget(sourceDir, restoreDir)));
+        final var realRestorePath = restoreTargets.mapToRestorePath(sourceDir);
+        final var restoredAPng = realRestorePath.resolve(aPng.getFileName().toString());
+        final var restoredBPng = realRestorePath.resolve(bPng.getFileName().toString());
+        final var restoredCPng = realRestorePath.resolve(cPng.getFileName().toString());
+        final var restoredFolder = realRestorePath.resolve("folder");
+        final var restoredExternal = realRestorePath.resolve("external.png");
+
+        final var metadataParser = FileMetadataParserFactory.newInstance();
+
+        //when "A.png" is restored
+        underTest.execute(RestoreTask.builder()
+                .restoreTargets(restoreTargets)
+                .threads(threads)
+                .dryRun(false)
+                .deleteFilesNotInBackup(true)
+                .includedPath(aPng)
+                .build());
+
+        //then nothing else exists
+        assertFileIsFullyRestored(aPng, restoredAPng, metadataParser, configuration);
+        Assertions.assertTrue(Files.notExists(restoredBPng, LinkOption.NOFOLLOW_LINKS));
+        Assertions.assertTrue(Files.notExists(restoredCPng, LinkOption.NOFOLLOW_LINKS));
+        Assertions.assertTrue(Files.notExists(restoredFolder, LinkOption.NOFOLLOW_LINKS));
+        Assertions.assertTrue(Files.notExists(restoredExternal, LinkOption.NOFOLLOW_LINKS));
+
+        //when the "folder" is restored
+        underTest.execute(RestoreTask.builder()
+                .restoreTargets(restoreTargets)
+                .threads(threads)
+                .dryRun(false)
+                .deleteFilesNotInBackup(true)
+                .includedPath(sourceDir.resolve("folder"))
+                .build());
+
+        //then both "A.png" and the full contents of the "folder" are restored
+        assertFileIsFullyRestored(aPng, restoredAPng, metadataParser, configuration);
+        Assertions.assertTrue(Files.notExists(restoredBPng, LinkOption.NOFOLLOW_LINKS));
+        Assertions.assertTrue(Files.notExists(restoredCPng, LinkOption.NOFOLLOW_LINKS));
+        final var restoredInternal = Files.readSymbolicLink(realRestorePath.resolve("folder/internal.png")).toAbsolutePath();
+        Assertions.assertEquals(sourceDir.relativize(aPng), realRestorePath.relativize(restoredInternal));
+        Assertions.assertTrue(Files.notExists(restoredExternal, LinkOption.NOFOLLOW_LINKS));
+        assertFileMetadataMatches(sourceFolder, restoredFolder, metadataParser, configuration);
+    }
+
     @ParameterizedTest
     @MethodSource("restoreParameterProvider")
     void testExecuteShouldRestoreFilesToDestinationWhenTargetFilesAlreadyExistWithDifferentContent(

diff --git a/...va/com/github/nagyesta/filebarj/core/restore/pipeline/RestorePipelineIntegrationTest.java b/...va/com/github/nagyesta/filebarj/core/restore/pipeline/RestorePipelineIntegrationTest.java
@@ -389,7 +389,7 @@ void testDeleteLeftOverFilesShouldThrowExceptionWhenCalledWithNullThreadPool() t
 
         //when
         Assertions.assertThrows(IllegalArgumentException.class,
-                () -> underTest.deleteLeftOverFiles(true, null));
+                () -> underTest.deleteLeftOverFiles(null, true, null));
 
         //then + exception
     }

diff --git a/file-barj-job/README.md b/file-barj-job/README.md
@@ -63,6 +63,7 @@ java -jar build/libs/file-barj-job.jar \
      --key-store keys.p12 \
      --key-alias alias \
      --at-epoch-seconds 123456 \
+     --include-path /original/path/filter \
      --threads 2
 ```
 

diff --git a/file-barj-job/src/main/java/com/github/nagyesta/filebarj/job/Controller.java b/file-barj-job/src/main/java/com/github/nagyesta/filebarj/job/Controller.java
@@ -129,6 +129,7 @@ protected void doRestore(final RestoreProperties properties) {
                 .threads(properties.getThreads())
                 .dryRun(properties.isDryRun())
                 .deleteFilesNotInBackup(properties.isDeleteFilesNotInBackup())
+                .includedPath(properties.getIncludedPath())
                 .build();
         new RestoreController(properties.getBackupSource(), properties.getPrefix(), kek, properties.getPointInTimeEpochSeconds())
                 .execute(restoreTask);

diff --git a/file-barj-job/src/main/java/com/github/nagyesta/filebarj/job/cli/CliRestoreParser.java b/file-barj-job/src/main/java/com/github/nagyesta/filebarj/job/cli/CliRestoreParser.java
@@ -9,6 +9,7 @@
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.Optional;
 
 /**
  * Parser class for the command line arguments of the restore task.
@@ -21,6 +22,7 @@ public class CliRestoreParser extends CliICommonBackupFileParser<RestoreProperti
     private static final String TARGET = "target-mapping";
     private static final String DELETE_MISSING = "delete-missing";
     private static final String AT_EPOCH_SECONDS = "at-epoch-seconds";
+    private static final String INCLUDED_PATH = "include-path";
 
     /**
      * Creates a new {@link CliRestoreParser} instance and sets the input arguments.
@@ -37,6 +39,9 @@ public CliRestoreParser(final String[] args, final Console console) {
             final var prefix = commandLine.getOptionValue(PREFIX);
             final var nowEpochSeconds = Instant.now().getEpochSecond() + "";
             final var atPointInTime = Long.parseLong(commandLine.getOptionValue(AT_EPOCH_SECONDS, nowEpochSeconds));
+            final var includedPath = Optional.ofNullable(commandLine.getOptionValue(INCLUDED_PATH))
+                    .map(Path::of)
+                    .orElse(null);
             final var targets = new HashMap<Path, Path>();
             if (commandLine.hasOption(TARGET)) {
                 final var mappings = commandLine.getOptionValues(TARGET);
@@ -60,6 +65,7 @@ public CliRestoreParser(final String[] args, final Console console) {
                     .prefix(prefix)
                     .targets(targets)
                     .pointInTimeEpochSeconds(atPointInTime)
+                    .includedPath(includedPath)
                     .build();
         });
     }
@@ -102,6 +108,15 @@ protected Options createOptions() {
                         .required(false)
                         .type(Long.class)
                         .desc("The date and time using UTC epoch seconds at which the content should be restored.")
+                        .build())
+                .addOption(Option.builder()
+                        .longOpt(INCLUDED_PATH)
+                        .numberOfArgs(1)
+                        .argName("path_from_backup")
+                        .required(false)
+                        .type(Path.class)
+                        .desc("Path of the file or directory which should be restored from the backup. Optional. If not provided,"
+                                + " all files should be restored.")
                         .build());
     }
 }
diff --git a/file-barj-job/src/main/java/com/github/nagyesta/filebarj/job/cli/RestoreProperties.java b/file-barj-job/src/main/java/com/github/nagyesta/filebarj/job/cli/RestoreProperties.java
@@ -21,4 +21,5 @@ public class RestoreProperties extends BackupFileProperties {
     private final boolean dryRun;
     private final boolean deleteFilesNotInBackup;
     private final long pointInTimeEpochSeconds;
+    private final Path includedPath;
 }