feat: add ReuseSnapshots option for btrfs sources

README.md

@@ -90,12 +90,16 @@ format and keys format.
 
 * [x] Fix a compatibility issue with MariaDB 11
 
-### 0.5
+### 0.5 (released)
 
-* [ ] Proxy support
+* [x] Proxy support
 
 ### 0.6
 
+* [ ] FTP/SFTP support
+
+### 0.7
+
 * [ ] Should be suitable for production
 
 ### 1.0

doc/src-btrfs.md

@@ -33,3 +33,13 @@ Optional, defaults: `[btrfs receive]`
 ### @DeleteCommand
 
 Optional, defaults: `[btrfs subvolume delete]`
+
+### @ReuseSnapshots
+
+Optional.
+
+Take a time interval (for example, `3d` for 3 days). If set, if there
+exists a snapshot that is more recent than that interval, then reuse
+that snapshot for creating a backup rather than creating a new one. This
+can be useful if you want to backup a single btrfs filesystem to two
+(or more) destinations.

sources/btrfs.go

@@ -30,6 +30,7 @@ type btrfsSource struct {
 	sendCommand     []string
 	receiveCommand  []string
 	deleteCommand   []string
+	reuseSnapshots  int
 }
 
 func newBtrfsSource(options *uback.Options) (uback.Source, error) {
@@ -48,13 +49,23 @@ func newBtrfsSource(options *uback.Options) (uback.Source, error) {
 		return nil, ErrTarPath
 	}
 
+	var reuseSnapshots int
+	if options.String["ReuseSnapshots"] != "" {
+		var err error
+		reuseSnapshots, err = uback.ParseInterval(options.String["ReuseSnapshots"])
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return &btrfsSource{
 		options:         options,
 		snapshotsPath:   snapshotsPath,
 		basePath:        basePath,
 		snapshotCommand: options.GetCommand("SnapshotCommand", []string{"btrfs", "subvolume", "snapshot"}),
 		sendCommand:     options.GetCommand("SendCommand", []string{"btrfs", "send"}),
 		deleteCommand:   options.GetCommand("DeleteCommand", []string{"btrfs", "subvolume", "delete"}),
+		reuseSnapshots:  reuseSnapshots,
 	}, nil
 }
 
@@ -104,20 +115,44 @@ func (s *btrfsSource) RemoveSnapshot(snapshot uback.Snapshot) error {
 // Part of uback.Source interface
 func (s *btrfsSource) CreateBackup(baseSnapshot *uback.Snapshot) (uback.Backup, io.ReadCloser, error) {
 	snapshot := time.Now().UTC().Format(uback.SnapshotTimeFormat)
-	tmpSnapshotPath := path.Join(s.snapshotsPath, fmt.Sprintf("_tmp-%s", snapshot))
 	finalSnapshotPath := path.Join(s.snapshotsPath, snapshot)
+	tmpSnapshotPath := path.Join(s.snapshotsPath, fmt.Sprintf("_tmp-%s", snapshot))
 
-	if s.snapshotsPath == "" {
-		baseSnapshot = nil
+	if s.reuseSnapshots != 0 {
+		snapshots, err := uback.SortedListSnapshots(s)
+		if err != nil {
+			return uback.Backup{}, nil, err
+		}
+
+		if len(snapshots) > 0 {
+			t, err := snapshots[0].Time()
+			if err != nil {
+				return uback.Backup{}, nil, err
+			}
+
+			if time.Now().UTC().Sub(t).Seconds() <= float64(s.reuseSnapshots) {
+				snapshot = string(snapshots[0])
+				finalSnapshotPath = path.Join(s.snapshotsPath, snapshot)
+				tmpSnapshotPath = finalSnapshotPath
+			}
+		}
 	}
 
-	err := uback.RunCommand(btrfsLog, uback.BuildCommand(s.snapshotCommand, "-r", s.basePath, tmpSnapshotPath))
-	if err != nil {
-		return uback.Backup{}, nil, err
+
+	if s.snapshotsPath == "" {
+		baseSnapshot = nil
 	}
 
 	backup := uback.Backup{Snapshot: uback.Snapshot(snapshot), BaseSnapshot: baseSnapshot}
-	btrfsLog.Printf("creating backup: %s", backup.Filename())
+	if tmpSnapshotPath != finalSnapshotPath {
+		err := uback.RunCommand(btrfsLog, uback.BuildCommand(s.snapshotCommand, "-r", s.basePath, tmpSnapshotPath))
+		if err != nil {
+			return uback.Backup{}, nil, err
+		}
+		btrfsLog.Printf("creating backup: %s", backup.Filename())
+	} else {
+		btrfsLog.Printf("reusing backup: %s", backup.Filename())
+	}
 
 	args := []string{}
 	if baseSnapshot != nil {
@@ -129,7 +164,7 @@ func (s *btrfsSource) CreateBackup(baseSnapshot *uback.Snapshot) (uback.Backup,
 			_ = uback.RunCommand(btrfsLog, uback.BuildCommand(s.deleteCommand, tmpSnapshotPath))
 			return err
 		}
-		if s.snapshotsPath != "" {
+		if tmpSnapshotPath != finalSnapshotPath {
 			return os.Rename(tmpSnapshotPath, finalSnapshotPath)
 		}
 		return nil

sources/proxy.go

@@ -101,7 +101,7 @@ func (s *proxySource) CreateBackup(baseSnapshot *uback.Snapshot) (uback.Backup,
 	}
 
 	var backup uback.Backup
-	err = rpcClient.Call("Source.CreateBackup", &CreateBackupArgs{Options: uback.ProxiedOptions(s.options)}, &backup)
+	err = rpcClient.Call("Source.CreateBackup", &CreateBackupArgs{Options: uback.ProxiedOptions(s.options), Snapshot: baseSnapshot}, &backup)
 	if err != nil {
 		return uback.Backup{}, nil, err
 	}

tests/src_btrfs_tests.py

@@ -1,5 +1,7 @@
 from .common import *
 
+import json
+
 class SrcBtrfsTests(unittest.TestCase, SrcBaseTests):
     def setUp(self):
         test_root = os.environ.get("UBACK_BTRFS_TEST_ROOT")
@@ -43,3 +45,27 @@ def test_btrfs_source(self):
             "send-command=sudo btrfs send,delete-command=sudo btrfs subvolume delete"
         dest = f"id=test,type=fs,path={self.tmpdir}/backups,@retention-policy=daily=3,key-file={self.tmpdir}/backup.key"
         self._test_src(self.tmpdir, source, dest, "receive-command=sudo btrfs receive", test_ignore=False, test_delete=True)
+
+    def test_btrfs_reuse_snapshots(self):
+        if self.tmpdir is None:
+            return
+
+        source = f"type=btrfs,path={self.tmpdir}/source,key-file={self.tmpdir}/backup.pub,state-file={self.tmpdir}/state.json,snapshots-path={self.tmpdir}/snapshots,full-interval=weekly," +\
+            "send-command=sudo btrfs send,delete-command=sudo btrfs subvolume delete,reuse-snapshots=1d"
+        dest = f"type=fs,@retention-policy=daily=3,key-file={self.tmpdir}/backup.key"
+
+        ensure_dir(f"{self.tmpdir}/backups1")
+        ensure_dir(f"{self.tmpdir}/backups2")
+        ensure_dir(f"{self.tmpdir}/restore")
+        ensure_dir(f"{self.tmpdir}/source")
+        subprocess.check_call([uback, "key", "gen", f"{self.tmpdir}/backup.key", f"{self.tmpdir}/backup.pub"])
+        with open(f"{self.tmpdir}/source/a", "w+") as fd: fd.write("av1")
+
+        b1 = subprocess.check_output([uback, "backup", source, f"id=test1,{dest},path={self.tmpdir}/backups1"]).strip().decode()
+        time.sleep(0.01)
+        b2 = subprocess.check_output([uback, "backup", source, f"id=test2,{dest},path={self.tmpdir}/backups2"]).strip().decode()
+        s = b1.split("-")[0]
+        self.assertEqual(b1, b2)
+        self.assertEqual(set(os.listdir(f"{self.tmpdir}/snapshots")), {s})
+        with open(f"{self.tmpdir}/state.json") as fd:
+            self.assertEqual(json.load(fd), {"test1": s, "test2": s})