feat: custom sources

.errcheck-exclude.cfg

@@ -2,3 +2,4 @@
 (*io.PipeWriter).CloseWithError
 encoding/binary.Write(*bytes.Buffer)
 (*github.com/spf13/cobra.Command).Help
+(*os.Process).Kill

README.md

@@ -27,10 +27,11 @@ source allow it) and retention policy.
 
 ## Quickstart and Documentation
 
-All the documentation is in the [doc/](doc/) directory. You should
-start by the [tutorial](doc/tutorial.md) and then jump to advanced
-topics ([file format](doc/file-format.md), TODO: custom sources, custom
-destinations) or the documentation specific to each source or destination.
+All the documentation is in the [doc/](doc/) directory. You should start
+by the [tutorial](doc/tutorial.md) and then jump to advanced topics ([file
+format](doc/file-format.md), [custom sources](doc/custom-sources.md),
+TODO: custom destinations) or the documentation specific to each source
+or destination.
 
 ## Supported Sources
 
@@ -64,7 +65,7 @@ stable with a good test suite. Here is a rough sketch of the roadmap :
 
 ### 0.2 (next)
 
-* [ ] Custom sources
+* [x] Custom sources
 * [ ] Custom destinations
 
 ### 0.3

cmd/backup.go

@@ -112,7 +112,7 @@ var (
 			}
 
 			pr, pw := io.Pipe()
-			cw, err := container.NewWriter(pw, &srcOpts.PublicKey, srcOpts.Options.String["Type"], compressionLevel)
+			cw, err := container.NewWriter(pw, &srcOpts.PublicKey, srcOpts.SourceType, compressionLevel)
 			if err != nil {
 				logrus.Fatal(err)
 			}

cmd/options.go

@@ -16,6 +16,7 @@ import (
 type optionsBuilder struct {
 	Options           *uback.Options
 	Source            uback.Source
+	SourceType        string
 	Destination       uback.Destination
 	RetentionPolicies []uback.RetentionPolicy
 	PrivateKey        x25519.PrivateKey
@@ -29,7 +30,7 @@ func newOptionsBuilder(options *uback.Options, err error) *optionsBuilder {
 
 func (o *optionsBuilder) WithSource() *optionsBuilder {
 	if o.Error == nil {
-		o.Source, o.Error = sources.New(o.Options)
+		o.Source, o.SourceType, o.Error = sources.New(o.Options)
 	}
 	return o
 }

destinations/fs.go

@@ -50,7 +50,7 @@ func (d *fsDestination) ListBackups() ([]uback.Backup, error) {
 			continue
 		}
 
-		backup, err := uback.ParseBackupFilename(entry.Name())
+		backup, err := uback.ParseBackupFilename(entry.Name(), true)
 		if err != nil {
 			fsLog.WithFields(logrus.Fields{
 				"file": entry.Name(),

destinations/object-storage.go

@@ -103,7 +103,7 @@ func (d *objectStorageDestination) ListBackups() ([]uback.Backup, error) {
 			continue
 		}
 
-		backup, err := uback.ParseBackupFilename(path.Base(obj.Key))
+		backup, err := uback.ParseBackupFilename(path.Base(obj.Key), true)
 		if err != nil {
 			osLog.WithFields(logrus.Fields{
 				"key": obj.Key,

doc/custom-sources.md

@@ -0,0 +1,112 @@
+# Custom Sources
+
+Custom sources are sources that are not built-in into `uback`, but
+implemented by an external command, typically a script, under the control
+of the user.
+
+## Custom Sources for Users
+
+When creating a backup, you give `command` as the source type, and set the
+`@SourceCommand` option to the custom source command. It can be either
+a full path or relative path if the custom source command in in the PATH.
+
+```shell
+$ uback backup type=command,@source-command=uback-tar-src,path=/etc,snapshots-path=/var/lib/uback/custom-tar-snapshots/ $DEST_OPTIONS
+```
+
+When restoring a backup, on the other hand, the custom command MUST be
+in the PATH, since the name of the custom source command will be extracted
+from the backup file header.
+
+## Custom Sources for Implementors
+
+The first argument passed to the command is the kind of command: `source`
+for a source, and `destination` for a destination. In the context of
+this document, the first argument will always be `source`.
+
+`uback` passes the requested operation and arguments as the next arguments
+on the command line, and options passed to the source as environment
+variable. For example, the `SnapshotsPath` will be translated into the
+`UBACK_OPT_SNAPSHOTS_PATH` environment variable. If the option is a
+slice option (for example `@AdditionalArguments`), it will be passed into
+`UBACK_SOPT_ADDITIONAL_ARGUMENTS`, and the value will be a JSON-serialized
+string array (the `S` in `SOPT` stands for "slice").
+
+A custom source command should follow the usual external process
+conventions : use stdout for normal output that will be consumed by
+`uback`, use stdin for normal input that will be given by `uback`,
+use stderr to print free-format messages destined to the end user. An
+exit code of 0 indicates a successful operation, whereas a non-zero one
+indicates a failure.
+
+A source must implement those operations, which will be described in
+the next sections :
+
+* type
+* list-snapshots
+* remove-snapshot
+* create-backup
+* restore-backup
+
+## Operations
+
+### type
+
+This operation takes no argument, and should just prints the type of the
+source, which will be used in the restoration process to determine what
+source will be responsible with restoring the backup.
+
+It should be `command:script-name`, where `script-name` is the normal name
+of the custom source command. If the `command:` prefix is not provided, it
+is automatically added (after emitting a warning) ; you can prevent that
+behavior by adding a `:` prefix (the `:` prefix will then be stripped).
+
+It is also a good place to validate the options.
+
+### list-snapshots
+
+This operation takes no argument, and should print all snapshots usable
+for creating incremental backups, one per line.
+
+`list-snapshots` is allowed to return invalid snapshots names as long
+as they start with a `.` or a `_`. Those entries will be ignored.
+
+### remove-snapshot
+
+This operation takes one argument, and should remove the snapshot
+identified by the argument.
+
+### create-backup
+
+This operation takes one optional argument, the base snapshot. If
+present, the source must try to create an incremental backup from the
+given snapshot. If not present, the source must create a new full backup.
+
+The source must first prints the name of the backup in a single line
+(`(snapshot)-full` for a full backup, `(snapshot)-from-(baseSnapshot)`
+for an incremental backup) and then just stream the backup data to stdout.
+
+### restore-backup
+
+Note that as a special cases, the options are not validated by the `type`
+operation before calling this operation: in a backup restoration, this
+operation is directly called.
+
+The first argument of the operation is the target directory, where to
+restore the backup.
+
+The second argument of the operation is the backup snapshot.
+
+The third argument of the operation is optional ; it is the backup base
+snapshot if the backup is an incremental one.
+
+If the backup is an incremental one, it is guaranteed that
+`restore-backup` has been called on the base just before.
+
+Backup data is passed to the custom source via stdin. No output is
+expected from the custom source.
+
+## Example
+
+As an example, you can look at the [uback-tar-src](../tests/uback-tar-src)
+test script, which reimplement the tar source as a bash script.

lib/utils.go

@@ -15,7 +15,7 @@ import (
 )
 
 var (
-	backupFilenameRe   = regexp.MustCompile(fmt.Sprintf(`^(%s)-(full|from-(%s))\.ubkp$`, SnapshotRe, SnapshotRe))
+	backupFilenameRe   = regexp.MustCompile(fmt.Sprintf(`^(%s)-(full|from-(%s))(\.ubkp)?$`, SnapshotRe, SnapshotRe))
 	SnapshotRe         = `\d{8}T\d{6}\.\d{3}`  // Regexp matching a snapshot name
 	SnapshotTimeFormat = "20060102T150405.000" // Time format of a snapshot, for time.Parse / time.Format
 )
@@ -102,13 +102,17 @@ func SortedListSnapshots(src Source) ([]Snapshot, error) {
 }
 
 // Reverse of Backup.Filename()
-func ParseBackupFilename(f string) (Backup, error) {
+func ParseBackupFilename(f string, requireExt bool) (Backup, error) {
 	f = path.Base(f)
 	m := backupFilenameRe.FindStringSubmatch(f)
 	if m == nil {
 		return Backup{}, fmt.Errorf("cannot parse backup filename: %s", f)
 	}
 
+	if requireExt && m[4] != ".ubkp" {
+		return Backup{}, fmt.Errorf("cannot parse backup filename: %s: missing or invalid extension '%s'", f, m[4])
+	}
+
 	if m[2] == "full" {
 		return Backup{Snapshot: Snapshot(m[1])}, nil
 	}
@@ -158,7 +162,11 @@ func WrapSourceCommand(backup Backup, cmd *exec.Cmd, finalize func(err error) er
 	}
 
 	go func() {
-		pw.CloseWithError(finalize(cmd.Wait()))
+		if finalize == nil {
+			pw.CloseWithError(cmd.Wait())
+		} else {
+			pw.CloseWithError(finalize(cmd.Wait()))
+		}
 	}()
 
 	return backup, pr, nil