feat: custom destinations

README.md

@@ -30,8 +30,8 @@ source allow it) and retention policy.
 All the documentation is in the [doc/](doc/) directory. You should start
 by the [tutorial](doc/tutorial.md) and then jump to advanced topics ([file
 format](doc/file-format.md), [custom sources](doc/custom-sources.md),
-TODO: custom destinations) or the documentation specific to each source
-or destination.
+[custom destinations](doc/custom-destinations.md)) or the documentation
+specific to each source or destination.
 
 ## Supported Sources
 
@@ -50,7 +50,7 @@ storage
 `uback` is in a preliminary stage, quite lacking feature-wide, but fairly
 stable with a good test suite. Here is a rough sketch of the roadmap :
 
-### 0.1 (released)
+### 0.1
 
 * [x] Core features:
   * [x] Backups & Incremental Backups
@@ -63,14 +63,16 @@ stable with a good test suite. Here is a rough sketch of the roadmap :
 * [x] Documentation
 * [x] CI/Release Management
 
-### 0.2 (next)
+### 0.2 (released)
 
 * [x] Custom sources
-* [ ] Custom destinations
+* [x] Custom destinations
 
-### 0.3
+### 0.3 (next)
 
-* [ ] Proxy support
+* [ ] switch to [age](https://age-encryption.org/) for encryption. This
+will be the first (and hopefully the last) breaking change for the file
+format and keys format.
 
 ### 0.4
 
@@ -79,6 +81,10 @@ stable with a good test suite. Here is a rough sketch of the roadmap :
 
 ### 0.5
 
+* [ ] Proxy support
+
+### 0.6
+
 * [ ] remove mariabackup footguns
   * [ ] add the option to use a dockerized mariabackup in the restoration
   process to have an exect version match

destinations/command.go

@@ -0,0 +1,143 @@
+package destinations
+
+import (
+	"github.com/sloonz/uback/lib"
+
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/gobuffalo/flect"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	ErrCommandMissing = errors.New("command destination: missing command")
+	commandLog        = logrus.WithFields(logrus.Fields{
+		"destination": "fs",
+	})
+)
+
+type commandDestination struct {
+	options *uback.Options
+	command string
+	env     []string
+}
+
+func newCommandDestination(options *uback.Options) (uback.Destination, error) {
+	command := options.String["Command"]
+	if command == "" {
+		return nil, ErrCommandMissing
+	}
+
+	env := os.Environ()
+	for k, v := range options.String {
+		env = append(env, fmt.Sprintf("UBACK_OPT_%s=%s", flect.New(k).Underscore().ToUpper().String(), v))
+	}
+	for k, v := range options.StrSlice {
+		jsonVal, err := json.Marshal(v)
+		if err != nil {
+			return nil, err
+		}
+		env = append(env, fmt.Sprintf("UBACK_SOPT_%s=%s", flect.New(k).Underscore().ToUpper().String(), string(jsonVal)))
+	}
+
+	buf := bytes.NewBuffer(nil)
+	cmd := exec.Command(command, "destination", "validate-options")
+	cmd.Stdout = buf
+	cmd.Stderr = os.Stderr
+	cmd.Env = env
+	err := cmd.Run()
+	if err != nil {
+		return nil, err
+	}
+
+	return &commandDestination{options: options, command: command, env: env}, nil
+}
+
+func (d *commandDestination) ListBackups() ([]uback.Backup, error) {
+	var res []uback.Backup
+
+	buf := bytes.NewBuffer(nil)
+	cmd := exec.Command(d.command, "destination", "list-backups")
+	cmd.Stdout = buf
+	cmd.Stderr = os.Stderr
+	cmd.Env = d.env
+	err := cmd.Run()
+	if err != nil {
+		return nil, err
+	}
+
+	for {
+		entry, err := buf.ReadString('\n')
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+
+		entry = strings.TrimSpace(entry)
+		if entry == "" {
+			continue
+		}
+
+		if strings.HasPrefix(entry, ".") || strings.HasPrefix(entry, "_") {
+			continue
+		}
+
+		backup, err := uback.ParseBackupFilename(entry, false)
+		if err != nil {
+			commandLog.WithFields(logrus.Fields{
+				"entry": entry,
+			})
+			logrus.Warnf("invalid backup file: %v", err)
+			continue
+		}
+
+		res = append(res, backup)
+	}
+
+	return res, nil
+}
+
+func (d *commandDestination) RemoveBackup(backup uback.Backup) error {
+	cmd := exec.Command(d.command, "destination", "remove-backup", backup.FullName())
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+	cmd.Env = d.env
+	return cmd.Run()
+}
+
+func (d *commandDestination) SendBackup(backup uback.Backup, data io.Reader) error {
+	cmd := exec.Command(d.command, "destination", "send-backup", backup.FullName())
+	cmd.Stdin = data
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+	cmd.Env = d.env
+	return cmd.Run()
+}
+
+func (d *commandDestination) ReceiveBackup(backup uback.Backup) (io.ReadCloser, error) {
+	pr, pw := io.Pipe()
+	cmd := exec.Command(d.command, "destination", "receive-backup", backup.FullName())
+	cmd.Stdout = pw
+	cmd.Stderr = os.Stderr
+	cmd.Env = d.env
+
+	commandLog.Printf("running: %v", cmd.String())
+	err := cmd.Start()
+	if err != nil {
+		return nil, err
+	}
+
+	go func() {
+		pw.CloseWithError(cmd.Wait())
+	}()
+
+	return pr, nil
+}

destinations/new.go

@@ -12,6 +12,8 @@ func New(options *uback.Options) (uback.Destination, error) {
 		return newFSDestination(options)
 	case "object-storage":
 		return newObjectStorageDestination(options)
+	case "command":
+		return newCommandDestination(options)
 	default:
 		return nil, fmt.Errorf("invalid destination type %v", options.String["Type"])
 	}

doc/custom-destinations.md

@@ -0,0 +1,85 @@
+# Custom Destinations
+
+Custom destinations are destintations that are not build-in into `uback`,
+but implemented by an external command, typicall a script, under the
+control of the user.
+
+## Custom Destinations for Users
+
+When creating a backup, you give `command` as the destination type,
+and set the `Command` option to the custom source command. It can be
+either a full path or relative path if the custom destination command
+in in the PATH.
+
+```shell
+$ uback backup $SOURCE_OPTIONS id=test,type=custom,command=uback-fs-dest
+```
+
+## Custom Destinations for Implementors
+
+The first argument passed to the command is the kind of command: `source`
+for a source, and `destination` for a destination. In the context of
+this document, the first argument will always be `destination`.
+
+`uback` passes the requested operation and arguments as the next arguments
+on the command line, and options passed to the destination as environment
+variable. For example, the `SnapshotsPath` will be translated into the
+`UBACK_OPT_SNAPSHOTS_PATH` environment variable. If the option is a
+slice option (for example `@AdditionalArguments`), it will be passed into
+`UBACK_SOPT_ADDITIONAL_ARGUMENTS`, and the value will be a JSON-serialized
+string array (the `S` in `SOPT` stands for "slice").
+
+A custom destination command should follow the usual external process
+conventions : use stdout for normal output that will be consumed by
+`uback`, use stdin for normal input that will be given by `uback`,
+use stderr to print free-format messages destined to the end user. An
+exit code of 0 indicates a successful operation, whereas a non-zero one
+indicates a failure.
+
+A destination must implement those operations, which will be described in
+the next sections :
+
+* validate-options
+* list-backups
+* remove-backup
+* send-backup
+* receive-backup
+
+## Operations
+
+### validate-options
+
+This operation takes no argument ; it is called before any other operation
+and should be used to validate options.
+
+### list-backups
+
+This operation takes no argument, and should print all backups currently
+present on the destination, one per line. You can either
+give a backup name (`20210102T000000.000-full`) or a filename
+(`20210102T000000.000-full.ubkp`).
+
+For convenience purposes, lines starting with a `.` or `_` are ignored.
+
+### remove-backup
+
+This operation takes one argument, the full
+name of a backup (`20210102T000000.000-full` or
+`20210102T000000.000-from-20210101T000000.000`), and should remove the
+backup on the destination.
+
+### send-backup
+
+This operation takes one argument, the full name of a backup. `uback`
+will provide the backup to the command standard input ; the command
+should store it.
+
+### receive-backup
+
+This operation takes one argument, the full name of a backup. The command
+should output the backup on its standard output.
+
+## Example
+
+As an example, you can look at the [uback-fs-test](../tests/uback-fs-test)
+test script, which reimplement the fs destination as a bash script.

doc/custom-sources.md

@@ -6,9 +6,9 @@ of the user.
 
 ## Custom Sources for Users
 
-When creating a backup, you give `command` as the source type, and set the
-`@SourceCommand` option to the custom source command. It can be either
-a full path or relative path if the custom source command in in the PATH.
+When creating a backup, you give `command` as the source type, and set
+the `Command` option to the custom source command. It can be either a
+full path or relative path if the custom source command in in the PATH.
 
 ```shell
 $ uback backup type=command,@source-command=uback-tar-src,path=/etc,snapshots-path=/var/lib/uback/custom-tar-snapshots/ $DEST_OPTIONS

tests/dest-command.bats

@@ -0,0 +1,42 @@
+load "helpers/bats-support/load"
+load "helpers/bats-assert/load"
+
+UBACK="$BATS_TEST_DIRNAME/../uback"
+TEST_TMPDIR="$BATS_RUN_TMPDIR/dest-command"
+
+@test "command destination" {
+	export PATH="$PATH:$BATS_TEST_DIRNAME"
+	mkdir -p "$TEST_TMPDIR/backups"
+	$UBACK key gen "$TEST_TMPDIR/backup.key" "$TEST_TMPDIR/backup.pub"
+	source=type=tar,path="$TEST_TMPDIR/source",key-file="$TEST_TMPDIR/backup.pub",state-file="$TEST_TMPDIR/state.json",snapshots-path="$TEST_TMPDIR/snapshots",full-interval=weekly
+	dest=id=test,type=command,command=uback-fs-dest,path="$TEST_TMPDIR/backups",@retention-policy=daily=3,key-file="$TEST_TMPDIR/backup.key"
+
+	mkdir -p "$TEST_TMPDIR/restore"
+	mkdir -p "$TEST_TMPDIR/source"
+	echo "hello" > "$TEST_TMPDIR/source/a"
+
+	# Full 1
+	assert_equal "$($UBACK list backups "$dest" | wc -l)" 0
+	$UBACK backup -n -f "$source" "$dest"
+	assert_equal "$($UBACK list backups "$dest" | wc -l)" 1
+	sleep 0.01
+
+	# Full 2
+	$UBACK backup -n -f "$source" "$dest"
+	assert_equal "$($UBACK list backups "$dest" | wc -l)" 2
+	sleep 0.01
+
+	# Incremental
+	echo "world" > "$TEST_TMPDIR/source/b"
+	$UBACK backup -n "$source" "$dest"
+	assert_equal "$($UBACK list backups "$dest" | wc -l)" 3
+
+	# Prune (remove full 1)
+	$UBACK prune backups "$dest"
+	assert_equal "$($UBACK list backups "$dest" | wc -l)" 2
+
+	# Restore full 2 + incremental
+	$UBACK restore -d "$TEST_TMPDIR/restore" "$dest"
+	assert_equal "$(cat "$TEST_TMPDIR"/restore/*/a)" "hello"
+	assert_equal "$(cat "$TEST_TMPDIR"/restore/*/b)" "world"
+}

tests/uback-fs-dest

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+if [ "$1" != "destination" ] ; then
+  echo "Invalid kind" >&2
+  exit 1
+fi
+
+case "$2" in
+  validate-options)
+    if [ "$UBACK_OPT_PATH" = "" ] ; then
+      echo "Missing option: Path" >&2
+      exit 1
+    fi
+    mkdir -p -- "$UBACK_OPT_PATH"
+    ;;
+  list-backups)
+    ls -- "$UBACK_OPT_PATH"
+    ;;
+  remove-backup)
+    rm -f -- "$UBACK_OPT_PATH/$3.ubkp"
+    ;;
+  send-backup)
+    cat > "$UBACK_OPT_PATH/_tmp-$3.ubkp"
+    mv "$UBACK_OPT_PATH/_tmp-$3.ubkp" "$UBACK_OPT_PATH/$3.ubkp"
+    ;;
+  receive-backup)
+    cat "$UBACK_OPT_PATH/$3.ubkp"
+    ;;
+  *)
+    echo "Invalid operation: $2" >&2
+    exit 1
+esac
+
+exit 0