feat(backend)!: Add backend to config and use rustic_backend

Cargo.toml

@@ -52,6 +52,7 @@ rustdoc-args = ["--document-private-items", "--generate-link-to-definition"]
 
 [dependencies]
 abscissa_core = { workspace = true }
+rustic_backend = { workspace = true }
 rustic_core = { workspace = true }
 
 # errors
@@ -64,6 +65,7 @@ log = { workspace = true }
 # serialization
 serde = { workspace = true }
 serde_json = { workspace = true }
+serde_path_to_error = { workspace = true }
 serde_with = { workspace = true }
 
 # other dependencies
@@ -85,24 +87,30 @@ humantime = { workspace = true }
 indicatif = { workspace = true }
 itertools = { workspace = true }
 jemallocator-global = { version = "0.3.2", optional = true }
-mimalloc = { version = "0.1.39", default_features = false, optional = true }
+mimalloc = { version = "0.1.39", default-features = false, optional = true }
 rhai = { workspace = true }
 simplelog = { workspace = true }
 
 [dev-dependencies]
 abscissa_core = { workspace = true, features = ["testing"] }
 aho-corasick = { workspace = true }
 dircmp = { workspace = true }
+insta = { version = "1.34.0", features = ["toml"] }
 once_cell = { workspace = true }
 pretty_assertions = { workspace = true }
+rstest = { workspace = true }
 rustic_testing = { path = "crates/rustic_testing" }
 tempfile = { workspace = true }
-toml = { workspace = true }
+toml = { workspace = true, features = ["parse", "preserve_order"] }
+toml_edit = { workspace = true, features = ["serde"] }
 
 [target.'cfg(not(windows))'.dependencies]
 libc = "0.2.150"
 [workspace.dependencies]
-rustic_core = { version = "0.1.2", features = ["cli"] }
+rustic_core = { path = "../rustic_core/crates/core", features = ["cli"] }
+rustic_backend = { path = "../rustic_core/crates/backends" }
+# rustic_core = { version = "0.1.2", features = ["cli"] }
+
 abscissa_core = { version = "0.7.0", default-features = false, features = ["application"] }
 
 # logging
@@ -117,6 +125,8 @@ anyhow = "1"
 serde = { version = "1", features = ["serde_derive"] }
 serde_with = { version = "3.4", features = ["base64"] }
 serde_json = "1"
+serde_path_to_error = "0.1.14"
+toml_edit = { version = "0.21.0", features = ["serde"] }
 
 # other dependencies
 aho-corasick = "1.1.2"
@@ -146,7 +156,7 @@ quickcheck = "1"
 quickcheck_macros = "1"
 tempfile = "3.8"
 pretty_assertions = "1.4"
-toml = "0.8"
+toml = { version = "0.8", features = ["parse", "preserve_order"] }
 dircmp = "0.2"
 
 # cargo-binstall support
@@ -201,6 +211,11 @@ lto = true
 debug-assertions = false
 codegen-units = 1
 
+# Faster insta testing runs
+[profile.dev.package]
+insta = { opt-level = 3 }
+similar = { opt-level = 3 }
+
 # Allows quick RPM file generation, if "cargo-generate-rpm" is installed:
 #  cargo build --release; cargo generate-rpm
 # will result in a file like target/generate-rpm/rustic-rs-0.6.1-1.x86_64.rpm

config/README.md

@@ -64,28 +64,33 @@ rustic.
 config file as a possible source of errors if you encounter problems. They could
 possibly shadow other values that you have already set.
 
+### Backend Options
+
+| Attribute  | Description                           | Default Value | Example Value | Environment Variable |
+| ---------- | ------------------------------------- | ------------- | ------------- | -------------------- |
+| repository | The path to the repository. Required. | Not set       | "/tmp/rustic" | RUSTIC_REPOSITORY    |
+| repo-hot   | The path to the hot repository.       | Not set       |               | RUSTIC_REPO_HOT      |
+
+### Backend Options (Additional)
+
+| Attribute           | Description                                                            | Default Value | Example Value                  |
+| ------------------- | ---------------------------------------------------------------------- | ------------- | ------------------------------ |
+| post-create-command | Command to execute after creating a snapshot in the **local backend**. | Not set       | "par2create -qq -n1 -r5 %file" |
+| post-delete-command | Command to execute after deleting a snapshot in the **local backend**. | Not set       | "sh -c \"rm -f %file*.par2\""  |
+
 ### Repository Options
 
 | Attribute        | Description                                                | Default Value            | Example Value          | Environment Variable    |
 | ---------------- | ---------------------------------------------------------- | ------------------------ | ---------------------- | ----------------------- |
 | cache-dir        | Path to the cache directory.                               | ~/.cache/rustic/$REPO_ID | ~/.cache/my_own_cache/ | RUSTIC_CACHE_DIR        |
 | no-cache         | If true, disables caching.                                 | false                    |                        | RUSTIC_NO_CACHE         |
-| repository       | The path to the repository. Required.                      | Not set                  | "/tmp/rustic"          | RUSTIC_REPOSITORY       |
-| repo-hot         | The path to the hot repository.                            | Not set                  |                        | RUSTIC_REPO_HOT         |
 | password         | The password for the repository.                           | Not set                  | "mySecretPassword"     | RUSTIC_PASSWORD         |
 | password-file    | Path to a file containing the password for the repository. | Not set                  |                        | RUSTIC_PASSWORD_FILE    |
 | password-command | Command to retrieve the password for the repository.       | Not set                  |                        | RUSTIC_PASSWORD_COMMAND |
 | warm-up          | If true, warms up the repository by file access.           | false                    |                        |                         |
 | warm-up-command  | Command to warm up the repository.                         | Not set                  |                        |                         |
 | warm-up-wait     | The wait time for warming up the repository.               | Not set                  |                        |                         |
 
-### Repository Options (Additional)
-
-| Attribute           | Description                                                        | Default Value | Example Value                  |
-| ------------------- | ------------------------------------------------------------------ | ------------- | ------------------------------ |
-| post-create-command | Command to execute after creating a snapshot in the local backend. | Not set       | "par2create -qq -n1 -r5 %file" |
-| post-delete-command | Command to execute after deleting a snapshot in the local backend. | Not set       | "sh -c \"rm -f %file*.par2\""  |
-
 ### Snapshot-Filter Options
 
 | Attribute    | Description                           | Default Value   | Example Value |
@@ -160,17 +165,24 @@ source-individual section.
 **Note**: Copy-targets are simply repositories with the same defaults as within
 the repository section.
 
-| Attribute           | Description                                                            | Default Value            | Example Value          |
-| ------------------- | ---------------------------------------------------------------------- | ------------------------ | ---------------------- |
-| cache-dir           | Path to the cache directory for the target repository.                 | ~/.cache/rustic/$REPO_ID | ~/.cache/my_own_cache/ |
-| no-cache            | If true, disables caching for the target repository.                   | false                    |                        |
-| password            | The password for the target repository.                                | Not set                  |                        |
-| password-file       | Path to a file containing the password for the target repository.      | Not set                  |                        |
-| password-command    | Command to retrieve the password for the target repository.            | Not set                  |                        |
-| post-create-command | Command to execute after creating a snapshot in the target repository. | Not set                  |                        |
-| post-delete-command | Command to execute after deleting a snapshot in the target repository. | Not set                  |                        |
-| repository          | The path or URL to the target repository.                              | Not set                  |                        |
-| repo-hot            | The path or URL to the hot target repository.                          | Not set                  |                        |
-| warm-up             | If true, warms up the target repository by file access.                | Not set                  |                        |
-| warm-up-command     | Command to warm up the target repository.                              | Not set                  |                        |
-| warm-up-wait        | The wait time for warming up the target repository.                    | Not set                  |                        |
+#### Copy Target Backends
+
+| Attribute           | Description                                                            | Default Value | Example Value                  |
+| ------------------- | ---------------------------------------------------------------------- | ------------- | ------------------------------ |
+| repository          | The path or URL to the target repository.                              | Not set       |                                |
+| repo-hot            | The path or URL to the hot target repository.                          | Not set       |                                |
+| post-create-command | Command to execute after creating a snapshot in the **local backend**. | Not set       | "par2create -qq -n1 -r5 %file" |
+| post-delete-command | Command to execute after deleting a snapshot in the **local backend**. | Not set       | "sh -c \"rm -f %file*.par2\""  |
+
+#### Copy Target Options
+
+| Attribute        | Description                                                       | Default Value            | Example Value          |
+| ---------------- | ----------------------------------------------------------------- | ------------------------ | ---------------------- |
+| cache-dir        | Path to the cache directory for the target repository.            | ~/.cache/rustic/$REPO_ID | ~/.cache/my_own_cache/ |
+| no-cache         | If true, disables caching for the target repository.              | false                    |                        |
+| password         | The password for the target repository.                           | Not set                  |                        |
+| password-file    | Path to a file containing the password for the target repository. | Not set                  |                        |
+| password-command | Command to retrieve the password for the target repository.       | Not set                  |                        |
+| warm-up          | If true, warms up the target repository by file access.           | Not set                  |                        |
+| warm-up-command  | Command to warm up the target repository.                         | Not set                  |                        |
+| warm-up-wait     | The wait time for warming up the target repository.               | Not set                  |                        |

config/copy_example.toml

@@ -3,19 +3,32 @@
 # If the config file is named "copy_example.toml", run "rustic -P copy_example copy" to copy all snapshots.
 # See "rustic copy --help" for options how to select or filter snapshots to copy.
 
-# [repository] specified the source repository
-[repository]
+# [backend] specifies the backend for a repository
+[backend]
 repository = "/tmp/repo"
+
+# [repository] specifies the options for a repository
+[repository]
 password = "test"
 
-# you can specify multiple targets
-[[copy.targets]]
-repository = "/tmp/repo2"
+# you can specify multiple copy targets
+[[copy]]
+# First copy target
+[copy.options]
+# Optional, but one of the three password options must be set
+# either on CLI, in the config file or in the environment variables
 password = "test"
 no-cache = true
 
-[[copy.targets]]
-repository = "rclone:ovh:backup"
-repo-hot = "clone:ovh:backup-hot"
+[copy.backend]
+repository = "/tmp/repo2"
+
+[[copy]]
+# Second copy target
+[copy.options]
 password-file = "/root/key-rustic-ovh"
 cache-dir = "/var/lib/cache/rustic" # explicitly specify cache dir for remote repository
+
+[copy.backend]
+repository = "rclone|ovh:backup"
+repo-hot = "rclone|ovh:backup-hot"

config/full.toml

@@ -21,11 +21,22 @@ dry-run = false
 # This is only an example how to set an rclone env variable. Default: No variables are defined.
 RCLONE_XXX = "true"
 
-# Repository options: These options define which backend to use and which password to use.
-[repository]
+# Backend options: These options define which backend to use.
+[backend]
 repository = "/repo/rustic" # Must be set
 repo-hot = "/my/hot/repo" # Default: not set
-# one of the three password options must be set
+
+# Additional backend options - depending on backend. These can be only set in the config file.
+[backend.options]
+post-create-command = "par2create -qq -n1 -r5 %file" # Only local backend; Default: not set
+post-delete-command = "sh -c \"rm -f %file*.par2\"" # Only local backend; Default: not set
+retry = "default" # Only rest/rclone backend; Allowed values: "false"/"off", "default" or number of retries
+timeout = "10min" # Only rest/rclone backend
+
+# Repository options: These options define which repository settings and passwords to use.
+[repository]
+# Optional, but one of the three password options must be set
+# either on CLI, in the config file or in environment variables
 password = "mySecretPassword"
 password-file = "/my/password.txt"
 password-command = "my_command.sh"
@@ -36,13 +47,6 @@ warm-up = false
 warm-up-command = "warmup.sh %id" # Default: not set
 warm-up-wait = "10min" # Default: not set
 
-# Additional repository options - depending on backend. These can be only set in the config file.
-[repository.options]
-post-create-command = "par2create -qq -n1 -r5 %file" # Only local backend; Default: not set
-post-delete-command = "sh -c \"rm -f %file*.par2\"" # Only local backend; Default: not set
-retry = "default" # Only rest/rclone backend; Allowed values: "false"/"off", "default" or number of retries
-timeout = "10min" # Only rest/rclone backend
-
 # Snapshot-filter options: These options apply to all commands that use snapshot filters
 [snapshot-filter]
 filter-host = ["host2", "host2"] # Default: no host filter
@@ -148,12 +152,14 @@ keep-withing-quarter-yearly = "0 year"
 keep-withing-half-yearly = "1 year"
 keep-within-yearly = "10 years"
 
-# Multiple targets are available for the copy command. Each specify a repository with exactly identical options as in
-# the [repository] section.
-[[copy.targets]]
-repository = "/repo/rustic" # Must be set
-repo-hot = "/my/hot/repo" # Default: not set
-# one of the three password options must be set
+# Multiple targets are available for the copy command.
+# Each specify a repository with exactly identical options as in
+# the [repository] and [backend] section.
+[[copy]]
+# First copy target
+[copy.options]
+# Optional, but one of the three password options must be set
+# either on CLI, in the config file or in the environment variables
 password = "mySecretPassword"
 password-file = "/my/password.txt"
 password-command = "my_command.sh"
@@ -164,6 +170,12 @@ warm-up = false
 warm-up-command = "warmup.sh %id" # Default: not set
 warm-up-wait = "10min" # Default: not set
 
-[[copy.targets]]
+[copy.backend]
+repository = "/repo/rustic" # Must be set
+repo-hot = "/my/hot/repo" # Default: not set
+
+[[copy]]
+# Second copy target
+[copy.backend]
 repository = "/repo/rustic2" # Must be set
 # ...

config/local.toml

@@ -2,9 +2,13 @@
 #
 # backup usage: "rustic -P local backup
 # cleanup:      "rustic -P local forget --prune
-#
-[repository]
+
+# Backend options: These options define which backend to use.
+[backend]
 repository = "/backup/rustic"
+
+# Repository options: These options define which repository settings and passwords to use.
+[repository]
 password-file = "/root/key-rustic"
 no-cache = true # no cache needed for local repository
 

config/ovh-hot-cold.toml

@@ -3,10 +3,12 @@
 #
 # backup usage: "rustic -P ovh-hot-cold backup
 # cleanup:      "rustic -P ovh-hot-cold forget --prune
-#
-[repository]
+
+[backend]
 repository = "rclone:ovh:backup-home"
 repo-hot = "rclone:ovh:backup-home-hot"
+
+[repository]
 password-file = "/root/key-rustic-ovh"
 cache-dir = "/var/lib/cache/rustic" # explicitly specify cache dir for remote repository
 warm-up = true # cold storage needs warm-up, just trying to access a file is sufficient to start the warm-up

config/par2.toml

@@ -2,14 +2,17 @@
 # error correction files using par2create to a local repository.
 # The commands can use the variable %file, %type and %id which are replaced by the filename, the
 # file type and the file id before calling the command.
-[repository]
+[backend]
 repository = "/tmp/repo"
-password = "test"
 
-[repository.options]
+[backend.options]
 # after saving a file in the repo, this command is called
 post-create-command = "par2create -qq -n1 -r5 %file"
 
 # after removing a file from the repo, this command is called.
 # Note that we want to use a "*" in the rm command, hence we have to call sh to resolve the wildcard!
 post-delete-command = "sh -c \"rm -f %file*.par2\""
+
+# Repository options: These options define which repository settings and passwords to use.
+[repository]
+password = "test"

config/rustic.toml

@@ -10,9 +10,12 @@
 log-level = "debug"
 log-file = "/log/rustic.log"
 
-# repository options: These options define which backend to use and which password to use.
-[repository]
+# Backend options: These options define which backend to use.
+[backend]
 repository = "/tmp/rustic"
+
+# Repository options: These options define which backend to use and which password to use.
+[repository]
 password = "mySecretPassword"
 
 # snapshot-filter options: These options apply to all commands that use snapshot filters

config/simple.toml

@@ -1,3 +1,7 @@
-[repository]
+# Backend options: These options define which backend to use.
+[backend]
 repository = "/tmp/repo"
+
+# Repository options: These options define which repository settings and passwords to use.
+[repository]
 password = "test"

src/commands.rs

@@ -244,7 +244,10 @@ impl Configurable<RusticConfig> for EntryPoint {
 /// [`RepositoryErrorKind::FromSplitError`]: crate::error::RepositoryErrorKind::FromSplitError
 fn open_repository(config: &Arc<RusticConfig>) -> Result<Repository<ProgressOptions, OpenStatus>> {
     let po = config.global.progress_options;
-    let repo = Repository::new_with_progress(&config.repository, po)?;
+
+    let backends = config.backend.to_backends()?;
+
+    let repo = Repository::new_with_progress(&config.repository, backends, po)?;
     match repo.password()? {
         // if password is given, directly return the result of find_key_in_backend and don't retry
         Some(pass) => {

src/commands/backup.rs

@@ -130,9 +130,10 @@ impl Runnable for BackupCmd {
 impl BackupCmd {
     fn inner_run(&self) -> Result<()> {
         let config = RUSTIC_APP.config();
+        let backends = config.backend.to_backends()?;
 
         let po = config.global.progress_options;
-        let repo = Repository::new_with_progress(&config.repository, po)?;
+        let repo = Repository::new_with_progress(&config.repository, backends, po)?;
         // Initialize repository if --init is set and it is not yet initialized
         let repo = if self.init && repo.config_id()?.is_none() {
             if config.global.dry_run {
@@ -224,7 +225,7 @@ impl BackupCmd {
                 .ignore_save_opts(opts.ignore_save_opts)
                 .ignore_filter_opts(opts.ignore_filter_opts)
                 .dry_run(config.global.dry_run);
-            let snap = repo.backup(&backup_opts, source.clone(), opts.snap_opts.to_snapshot()?)?;
+            let snap = repo.backup(&backup_opts, &source, opts.snap_opts.to_snapshot()?)?;
 
             if opts.json {
                 let mut stdout = std::io::stdout();