From 88c7f45ccb87d6ecb9a735ce8f1ef86b3b245b49 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Tue, 9 Jan 2024 20:23:22 -0700 Subject: [PATCH] Add --include and --remove flags Add new flags --include and --remove for backup command to support delta backups. --include is used when specific backup paths need to be scanned without scanning the whole backup set. --remove is used for removing backed up paths from the backup set. --- cmd/restic/cmd_backup.go | 56 ++++--- cmd/restic/cmd_restore.go | 2 +- internal/archiver/archiver.go | 237 +++++++++++++++-------------- internal/archiver/archiver_test.go | 12 +- internal/archiver/scanner.go | 24 +-- internal/archiver/scanner_test.go | 7 +- internal/fs/helpers.go | 29 +++- 7 files changed, 214 insertions(+), 153 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index a2b81a75954..d2277f4848e 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -21,6 +21,7 @@ import ( "github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -95,6 +96,8 @@ type BackupOptions struct { ExcludeIfPresent []string ExcludeCaches bool ExcludeLargerThan string + Includes []string + Removes []string Stdin bool StdinFilename string StdinCommand bool @@ -133,6 +136,8 @@ func init() { f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)") f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`) f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)") + f.StringArrayVarP(&backupOptions.Includes, "include", "i", nil, "include a `pattern` for folders/files to be scanned for the backup. This prevents a full scan and directly chooses files and folders specified in --include.") + f.StringArrayVar(&backupOptions.Removes, "remove", nil, "include a `pattern` for folders/files (backupsets) to be removed from the backups though they are still present on the file system. For removing inner files/folder use --exclude instead.") f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin") f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin") f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "execute command and store its stdout") @@ -165,15 +170,18 @@ func init() { // filterExisting returns a slice of all existing items, or an error if no // items exist at all. -func filterExisting(items []string) (result []string, err error) { +func filterExisting(items []string, removes []string) (result []string, err error) { for _, item := range items { - _, err := fs.Lstat(item) - if errors.Is(err, os.ErrNotExist) { - Warnf("%v does not exist, skipping\n", item) - continue - } + //This is where we are ensuring backupsets are removed when specified with --remove flag + if !fs.IsPathRemoved(removes, item) { + _, err := fs.Lstat(item) + if errors.Is(err, os.ErrNotExist) { + Warnf("%v does not exist, skipping\n", item) + continue + } - result = append(result, item) + result = append(result, item) + } } if len(result) == 0 { @@ -288,6 +296,16 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error { } } } + if len(opts.Includes) > 0 { + if err := filter.ValidatePatterns(opts.Includes); err != nil { + return errors.Fatalf("--include: %s", err) + } + } + if len(opts.Removes) > 0 { + if err := filter.ValidatePatterns(opts.Removes); err != nil { + return errors.Fatalf("--remove: %s", err) + } + } if opts.Stdin || opts.StdinCommand { if len(opts.FilesFrom) > 0 { @@ -425,7 +443,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er return nil, errors.Fatal("nothing to backup, please specify target files/dirs") } - targets, err = filterExisting(targets) + targets, err = filterExisting(targets, opts.Removes) if err != nil { return nil, err } @@ -620,6 +638,16 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter cancelCtx, cancel := context.WithCancel(wgCtx) defer cancel() + snapshotOpts := archiver.SnapshotOptions{ + Excludes: opts.Excludes, + Includes: opts.Includes, + Tags: opts.Tags.Flatten(), + Time: timeStamp, + Hostname: opts.Host, + ParentSnapshot: parentSnapshot, + ProgramVersion: "restic " + version, + } + if !opts.NoScan { sc := archiver.NewScanner(targetFS) sc.SelectByName = selectByNameFilter @@ -630,7 +658,8 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter if !gopts.JSON { progressPrinter.V("start scan on %v", targets) } - wg.Go(func() error { return sc.Scan(cancelCtx, targets) }) + + wg.Go(func() error { return sc.Scan(cancelCtx, targets, snapshotOpts) }) } arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency}) @@ -661,15 +690,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter arch.ChangeIgnoreFlags |= archiver.ChangeIgnoreCtime } - snapshotOpts := archiver.SnapshotOptions{ - Excludes: opts.Excludes, - Tags: opts.Tags.Flatten(), - Time: timeStamp, - Hostname: opts.Host, - ParentSnapshot: parentSnapshot, - ProgramVersion: "restic " + version, - } - if !gopts.JSON { progressPrinter.V("start backup on %v", targets) } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 6045a5d4133..b23c7ae2966 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -235,7 +235,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, matchedInsensitive, childMayMatchInsensitive, err := filter.ListWithChild(insensitiveIncludePatterns, strings.ToLower(item)) if err != nil { - msg.E("error for iexclude pattern: %v", err) + msg.E("error for iinclude pattern: %v", err) } selectedForRestore = matched || matchedInsensitive diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index e2f22ebeafc..c921d46f96b 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -214,7 +214,7 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error { // SaveDir stores a directory in the repo and returns the node. snPath is the // path within the current snapshot. -func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) { +func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc, opts SnapshotOptions) (d FutureNode, err error) { debug.Log("%v %v", snPath, dir) treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi) @@ -240,7 +240,7 @@ func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi pathname := arch.FS.Join(dir, name) oldNode := previous.Find(name) snItem := join(snPath, name) - fn, excluded, err := arch.Save(ctx, snItem, pathname, oldNode) + fn, excluded, err := arch.Save(ctx, snItem, pathname, oldNode, opts) // return error early if possible if err != nil { @@ -331,7 +331,7 @@ func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool { // Errors and completion needs to be handled by the caller. // // snPath is the path within the current snapshot. -func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) { +func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous *restic.Node, opts SnapshotOptions) (fn FutureNode, excluded bool, err error) { start := time.Now() debug.Log("%v target %q, previous %v", snPath, target, previous) @@ -346,139 +346,146 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous return FutureNode{}, true, nil } - // get file info and run remaining select functions that require file information - fi, err := arch.FS.Lstat(target) - if err != nil { - debug.Log("lstat() for %v returned error: %v", target, err) - err = arch.error(abstarget, err) + if !fs.IsPathIncluded(opts.Includes, target) { + fn = newFutureNodeWithResult(futureNodeResult{ + snPath: snPath, + target: target, + node: previous, + }) + } else { + // get file info and run remaining select functions that require file information + fi, err := arch.FS.Lstat(target) if err != nil { - return FutureNode{}, false, errors.WithStack(err) + debug.Log("lstat() for %v returned error: %v", target, err) + err = arch.error(abstarget, err) + if err != nil { + return FutureNode{}, false, errors.WithStack(err) + } + return FutureNode{}, true, nil + } + if !arch.Select(abstarget, fi) { + debug.Log("%v is excluded", target) + return FutureNode{}, true, nil } - return FutureNode{}, true, nil - } - if !arch.Select(abstarget, fi) { - debug.Log("%v is excluded", target) - return FutureNode{}, true, nil - } - switch { - case fs.IsRegularFile(fi): - debug.Log(" %v regular file", target) - - // check if the file has not changed before performing a fopen operation (more expensive, specially - // in network filesystems) - if previous != nil && !fileChanged(fi, previous, arch.ChangeIgnoreFlags) { - if arch.allBlobsPresent(previous) { - debug.Log("%v hasn't changed, using old list of blobs", target) - arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start)) - arch.CompleteBlob(previous.Size) - node, err := arch.nodeFromFileInfo(snPath, target, fi) + switch { + case fs.IsRegularFile(fi): + debug.Log(" %v regular file", target) + + // check if the file has not changed before performing a fopen operation (more expensive, specially + // in network filesystems) + if previous != nil && !fileChanged(fi, previous, arch.ChangeIgnoreFlags) { + if arch.allBlobsPresent(previous) { + debug.Log("%v hasn't changed, using old list of blobs", target) + arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start)) + arch.CompleteBlob(previous.Size) + node, err := arch.nodeFromFileInfo(snPath, target, fi) + if err != nil { + return FutureNode{}, false, err + } + + // copy list of blobs + node.Content = previous.Content + + fn = newFutureNodeWithResult(futureNodeResult{ + snPath: snPath, + target: target, + node: node, + }) + return fn, false, nil + } + + debug.Log("%v hasn't changed, but contents are missing!", target) + // There are contents missing - inform user! + err := errors.Errorf("parts of %v not found in the repository index; storing the file again", target) + err = arch.error(abstarget, err) if err != nil { return FutureNode{}, false, err } - - // copy list of blobs - node.Content = previous.Content - - fn = newFutureNodeWithResult(futureNodeResult{ - snPath: snPath, - target: target, - node: node, - }) - return fn, false, nil } - debug.Log("%v hasn't changed, but contents are missing!", target) - // There are contents missing - inform user! - err := errors.Errorf("parts of %v not found in the repository index; storing the file again", target) - err = arch.error(abstarget, err) + // reopen file and do an fstat() on the open file to check it is still + // a file (and has not been exchanged for e.g. a symlink) + file, err := arch.FS.OpenFile(target, fs.O_RDONLY|fs.O_NOFOLLOW, 0) if err != nil { - return FutureNode{}, false, err + debug.Log("Openfile() for %v returned error: %v", target, err) + err = arch.error(abstarget, err) + if err != nil { + return FutureNode{}, false, errors.WithStack(err) + } + return FutureNode{}, true, nil } - } - // reopen file and do an fstat() on the open file to check it is still - // a file (and has not been exchanged for e.g. a symlink) - file, err := arch.FS.OpenFile(target, fs.O_RDONLY|fs.O_NOFOLLOW, 0) - if err != nil { - debug.Log("Openfile() for %v returned error: %v", target, err) - err = arch.error(abstarget, err) + fi, err = file.Stat() if err != nil { - return FutureNode{}, false, errors.WithStack(err) + debug.Log("stat() on opened file %v returned error: %v", target, err) + _ = file.Close() + err = arch.error(abstarget, err) + if err != nil { + return FutureNode{}, false, errors.WithStack(err) + } + return FutureNode{}, true, nil } - return FutureNode{}, true, nil - } - fi, err = file.Stat() - if err != nil { - debug.Log("stat() on opened file %v returned error: %v", target, err) - _ = file.Close() - err = arch.error(abstarget, err) - if err != nil { - return FutureNode{}, false, errors.WithStack(err) + // make sure it's still a file + if !fs.IsRegularFile(fi) { + err = errors.Errorf("file %v changed type, refusing to archive", fi.Name()) + _ = file.Close() + err = arch.error(abstarget, err) + if err != nil { + return FutureNode{}, false, err + } + return FutureNode{}, true, nil } - return FutureNode{}, true, nil - } - // make sure it's still a file - if !fs.IsRegularFile(fi) { - err = errors.Errorf("file %v changed type, refusing to archive", fi.Name()) - _ = file.Close() - err = arch.error(abstarget, err) + // Save will close the file, we don't need to do that + fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() { + arch.StartFile(snPath) + }, func() { + arch.CompleteItem(snPath, nil, nil, ItemStats{}, 0) + }, func(node *restic.Node, stats ItemStats) { + arch.CompleteItem(snPath, previous, node, stats, time.Since(start)) + }) + + case fi.IsDir(): + debug.Log(" %v dir", target) + + snItem := snPath + "/" + oldSubtree, err := arch.loadSubtree(ctx, previous) + if err != nil { + err = arch.error(abstarget, err) + } if err != nil { return FutureNode{}, false, err } - return FutureNode{}, true, nil - } - // Save will close the file, we don't need to do that - fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() { - arch.StartFile(snPath) - }, func() { - arch.CompleteItem(snPath, nil, nil, ItemStats{}, 0) - }, func(node *restic.Node, stats ItemStats) { - arch.CompleteItem(snPath, previous, node, stats, time.Since(start)) - }) + fn, err = arch.SaveDir(ctx, snPath, target, fi, oldSubtree, + func(node *restic.Node, stats ItemStats) { + arch.CompleteItem(snItem, previous, node, stats, time.Since(start)) + }, opts) + if err != nil { + debug.Log("SaveDir for %v returned error: %v", snPath, err) + return FutureNode{}, false, err + } - case fi.IsDir(): - debug.Log(" %v dir", target) + case fi.Mode()&os.ModeSocket > 0: + debug.Log(" %v is a socket, ignoring", target) + return FutureNode{}, true, nil - snItem := snPath + "/" - oldSubtree, err := arch.loadSubtree(ctx, previous) - if err != nil { - err = arch.error(abstarget, err) - } - if err != nil { - return FutureNode{}, false, err - } + default: + debug.Log(" %v other", target) - fn, err = arch.SaveDir(ctx, snPath, target, fi, oldSubtree, - func(node *restic.Node, stats ItemStats) { - arch.CompleteItem(snItem, previous, node, stats, time.Since(start)) + node, err := arch.nodeFromFileInfo(snPath, target, fi) + if err != nil { + return FutureNode{}, false, err + } + fn = newFutureNodeWithResult(futureNodeResult{ + snPath: snPath, + target: target, + node: node, }) - if err != nil { - debug.Log("SaveDir for %v returned error: %v", snPath, err) - return FutureNode{}, false, err } - - case fi.Mode()&os.ModeSocket > 0: - debug.Log(" %v is a socket, ignoring", target) - return FutureNode{}, true, nil - - default: - debug.Log(" %v other", target) - - node, err := arch.nodeFromFileInfo(snPath, target, fi) - if err != nil { - return FutureNode{}, false, err - } - fn = newFutureNodeWithResult(futureNodeResult{ - snPath: snPath, - target: target, - node: node, - }) } - debug.Log("return after %.3f", time.Since(start).Seconds()) return fn, false, nil @@ -537,7 +544,7 @@ func (arch *Archiver) statDir(dir string) (os.FileInfo, error) { // SaveTree stores a Tree in the repo, returned is the tree. snPath is the path // within the current snapshot. -func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) { +func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc, opts SnapshotOptions) (FutureNode, int, error) { var node *restic.Node if snPath != "/" { @@ -575,7 +582,7 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, // this is a leaf node if subatree.Leaf() { - fn, excluded, err := arch.Save(ctx, join(snPath, name), subatree.Path, previous.Find(name)) + fn, excluded, err := arch.Save(ctx, join(snPath, name), subatree.Path, previous.Find(name), opts) if err != nil { err = arch.error(subatree.Path, err) @@ -611,7 +618,7 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, // not a leaf node, archive subtree fn, _, err := arch.SaveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) { arch.CompleteItem(snItem, oldNode, n, is, time.Since(start)) - }) + }, opts) if err != nil { return FutureNode{}, 0, err } @@ -678,6 +685,8 @@ type SnapshotOptions struct { Tags restic.TagList Hostname string Excludes []string + Includes []string + Removes []string Time time.Time ParentSnapshot *restic.Snapshot ProgramVersion string @@ -754,7 +763,7 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps debug.Log("starting snapshot") fn, nodeCount, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(n *restic.Node, is ItemStats) { arch.CompleteItem("/", nil, nil, is, time.Since(start)) - }) + }, opts) if err != nil { return err } diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index c6daed5bb50..8c78f5cbebd 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -227,7 +227,7 @@ func TestArchiverSave(t *testing.T) { } arch.runWorkers(ctx, wg) - node, excluded, err := arch.Save(ctx, "/", filepath.Join(tempdir, "file"), nil) + node, excluded, err := arch.Save(ctx, "/", filepath.Join(tempdir, "file"), nil, SnapshotOptions{Time: time.Now()}) if err != nil { t.Fatal(err) } @@ -304,7 +304,7 @@ func TestArchiverSaveReaderFS(t *testing.T) { } arch.runWorkers(ctx, wg) - node, excluded, err := arch.Save(ctx, "/", filename, nil) + node, excluded, err := arch.Save(ctx, "/", filename, nil, SnapshotOptions{Time: time.Now()}) t.Logf("Save returned %v %v", node, err) if err != nil { t.Fatal(err) @@ -845,7 +845,7 @@ func TestArchiverSaveDir(t *testing.T) { t.Fatal(err) } - ft, err := arch.SaveDir(ctx, "/", test.target, fi, nil, nil) + ft, err := arch.SaveDir(ctx, "/", test.target, fi, nil, nil, SnapshotOptions{Time: time.Now()}) if err != nil { t.Fatal(err) } @@ -918,7 +918,7 @@ func TestArchiverSaveDirIncremental(t *testing.T) { t.Fatal(err) } - ft, err := arch.SaveDir(ctx, "/", tempdir, fi, nil, nil) + ft, err := arch.SaveDir(ctx, "/", tempdir, fi, nil, nil, SnapshotOptions{Time: time.Now()}) if err != nil { t.Fatal(err) } @@ -1107,7 +1107,7 @@ func TestArchiverSaveTree(t *testing.T) { t.Fatal(err) } - fn, _, err := arch.SaveTree(ctx, "/", atree, nil, nil) + fn, _, err := arch.SaveTree(ctx, "/", atree, nil, nil, SnapshotOptions{Time: time.Now()}) if err != nil { t.Fatal(err) } @@ -2236,7 +2236,7 @@ func TestRacyFileSwap(t *testing.T) { arch.runWorkers(ctx, wg) // fs.Track will panic if the file was not closed - _, excluded, err := arch.Save(ctx, "/", tempfile, nil) + _, excluded, err := arch.Save(ctx, "/", tempfile, nil, SnapshotOptions{Time: time.Now()}) if err == nil { t.Errorf("Save() should have failed") } diff --git a/internal/archiver/scanner.go b/internal/archiver/scanner.go index 6ce2a47000b..e2154b2b777 100644 --- a/internal/archiver/scanner.go +++ b/internal/archiver/scanner.go @@ -38,7 +38,7 @@ type ScanStats struct { Bytes uint64 } -func (s *Scanner) scanTree(ctx context.Context, stats ScanStats, tree Tree) (ScanStats, error) { +func (s *Scanner) scanTree(ctx context.Context, stats ScanStats, tree Tree, opts SnapshotOptions) (ScanStats, error) { // traverse the path in the file system for all leaf nodes if tree.Leaf() { abstarget, err := s.FS.Abs(tree.Path) @@ -46,7 +46,7 @@ func (s *Scanner) scanTree(ctx context.Context, stats ScanStats, tree Tree) (Sca return ScanStats{}, err } - stats, err = s.scan(ctx, stats, abstarget) + stats, err = s.scan(ctx, stats, abstarget, opts) if err != nil { return ScanStats{}, err } @@ -57,7 +57,7 @@ func (s *Scanner) scanTree(ctx context.Context, stats ScanStats, tree Tree) (Sca // otherwise recurse into the nodes in a deterministic order for _, name := range tree.NodeNames() { var err error - stats, err = s.scanTree(ctx, stats, tree.Nodes[name]) + stats, err = s.scanTree(ctx, stats, tree.Nodes[name], opts) if err != nil { return ScanStats{}, err } @@ -72,7 +72,7 @@ func (s *Scanner) scanTree(ctx context.Context, stats ScanStats, tree Tree) (Sca // Scan traverses the targets. The function Result is called for each new item // found, the complete result is also returned by Scan. -func (s *Scanner) Scan(ctx context.Context, targets []string) error { +func (s *Scanner) Scan(ctx context.Context, targets []string, opts SnapshotOptions) error { debug.Log("start scan for %v", targets) cleanTargets, err := resolveRelativeTargets(s.FS, targets) @@ -88,7 +88,7 @@ func (s *Scanner) Scan(ctx context.Context, targets []string) error { return err } - stats, err := s.scanTree(ctx, ScanStats{}, *tree) + stats, err := s.scanTree(ctx, ScanStats{}, *tree, opts) if err != nil { return err } @@ -98,13 +98,13 @@ func (s *Scanner) Scan(ctx context.Context, targets []string) error { return nil } -func (s *Scanner) scan(ctx context.Context, stats ScanStats, target string) (ScanStats, error) { +func (s *Scanner) scan(ctx context.Context, stats ScanStats, target string, opts SnapshotOptions) (ScanStats, error) { if ctx.Err() != nil { return stats, nil } // exclude files by path before running stat to reduce number of lstat calls - if !s.SelectByName(target) { + if !s.SelectByName(target) || fs.IsPathRemoved(opts.Removes, target) { return stats, nil } @@ -131,9 +131,13 @@ func (s *Scanner) scan(ctx context.Context, stats ScanStats, target string) (Sca sort.Strings(names) for _, name := range names { - stats, err = s.scan(ctx, stats, filepath.Join(target, name)) - if err != nil { - return stats, err + var fullPath = filepath.Join(target, name) + + if fs.IsPathIncluded(opts.Includes, fullPath) { + stats, err = s.scan(ctx, stats, fullPath, opts) + if err != nil { + return stats, err + } } } stats.Dirs++ diff --git a/internal/archiver/scanner_test.go b/internal/archiver/scanner_test.go index 1b4cd1f7f2e..7b5f4aebd73 100644 --- a/internal/archiver/scanner_test.go +++ b/internal/archiver/scanner_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "runtime" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/restic/restic/internal/fs" @@ -112,7 +113,7 @@ func TestScanner(t *testing.T) { results[p] = s } - err = sc.Scan(ctx, []string{"."}) + err = sc.Scan(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) if err != nil { t.Fatal(err) } @@ -263,7 +264,7 @@ func TestScannerError(t *testing.T) { } } - err = sc.Scan(ctx, []string{"."}) + err = sc.Scan(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) if err != nil { t.Fatal(err) } @@ -310,7 +311,7 @@ func TestScannerCancel(t *testing.T) { } } - err = sc.Scan(ctx, []string{"."}) + err = sc.Scan(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) if err != nil { t.Errorf("unexpected error %v found", err) } diff --git a/internal/fs/helpers.go b/internal/fs/helpers.go index 4dd1e0e7338..3e54ed3710d 100644 --- a/internal/fs/helpers.go +++ b/internal/fs/helpers.go @@ -1,6 +1,9 @@ package fs -import "os" +import ( + "os" + "strings" +) // IsRegularFile returns true if fi belongs to a normal file. If fi is nil, // false is returned. @@ -11,3 +14,27 @@ func IsRegularFile(fi os.FileInfo) bool { return fi.Mode()&os.ModeType == 0 } + +func IsPathIncluded(includes []string, path string) bool { + var result bool = len(includes) == 0 + if !result { + for _, x := range includes { + if strings.Contains(x, path) || strings.Contains(path, x) { + result = true + break + } + } + } + return result +} + +func IsPathRemoved(removes []string, path string) bool { + if len(removes) != 0 { + for _, x := range removes { + if strings.Contains(path, x) { + return true + } + } + } + return false +}