From c52cda8509ee91198c844f945f1a6f702829bd2e Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Wed, 12 Jul 2023 10:32:40 +0200 Subject: [PATCH] refactor backup command --- crates/rustic_core/examples/backup.rs | 32 +++ crates/rustic_core/src/archiver.rs | 17 +- crates/rustic_core/src/archiver/parent.rs | 52 +++-- crates/rustic_core/src/backend/ignore.rs | 17 +- crates/rustic_core/src/commands.rs | 1 + crates/rustic_core/src/commands/backup.rs | 221 ++++++++++++++++++ crates/rustic_core/src/commands/cat.rs | 6 +- crates/rustic_core/src/commands/dump.rs | 4 +- crates/rustic_core/src/commands/restore.rs | 12 +- crates/rustic_core/src/lib.rs | 2 +- .../rustic_core/src/repofile/snapshotfile.rs | 8 +- crates/rustic_core/src/repository.rs | 96 +++++--- src/commands/backup.rs | 164 +------------ src/commands/diff.rs | 2 +- 14 files changed, 395 insertions(+), 239 deletions(-) create mode 100644 crates/rustic_core/examples/backup.rs create mode 100644 crates/rustic_core/src/commands/backup.rs diff --git a/crates/rustic_core/examples/backup.rs b/crates/rustic_core/examples/backup.rs new file mode 100644 index 000000000..919b56622 --- /dev/null +++ b/crates/rustic_core/examples/backup.rs @@ -0,0 +1,32 @@ +//! `backup` example +use rustic_core::{BackupOpts, PathList, Repository, RepositoryOptions, SnapshotFile}; +use simplelog::{Config, LevelFilter, SimpleLogger}; + +fn main() { + // Display info logs + let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); + + // Open repository + let repo_opts = RepositoryOptions { + repository: Some("/tmp/repo".to_string()), + password: Some("test".to_string()), + ..Default::default() + }; + + let repo = Repository::new(&repo_opts) + .unwrap() + .open() + .unwrap() + .to_indexed_ids() + .unwrap(); + + let backup_opts = BackupOpts::default(); + let source = PathList::from_string(".", true).unwrap(); // true: sanitize the given string + let dry_run = false; + + let snap = repo + .backup(&backup_opts, source, SnapshotFile::default(), dry_run) + .unwrap(); + + println!("successfully created snapshot:\n{snap:#?}") +} diff --git a/crates/rustic_core/src/archiver.rs b/crates/rustic_core/src/archiver.rs index 39ae0324f..a56332788 100644 --- a/crates/rustic_core/src/archiver.rs +++ b/crates/rustic_core/src/archiver.rs @@ -16,7 +16,6 @@ use crate::{ }, backend::{decrypt::DecryptWriteBackend, ReadSource, ReadSourceEntry}, blob::BlobType, - id::Id, index::{indexer::Indexer, indexer::SharedIndexer, IndexedBackend}, repofile::{configfile::ConfigFile, snapshotfile::SnapshotFile}, Progress, RusticResult, @@ -25,8 +24,7 @@ use crate::{ pub struct Archiver { file_archiver: FileArchiver, tree_archiver: TreeArchiver, - parent_tree: Option, - parent: Parent, + parent: Parent, indexer: SharedIndexer, be: BE, snap: SnapshotFile, @@ -37,22 +35,18 @@ impl Archiver { be: BE, index: I, config: &ConfigFile, - parent_tree: Option, - ignore_ctime: bool, - ignore_inode: bool, + parent: Parent, mut snap: SnapshotFile, ) -> RusticResult { let indexer = Indexer::new(be.clone()).into_shared(); - let mut summary = snap.summary.take().unwrap(); + let mut summary = snap.summary.take().unwrap_or_default(); summary.backup_start = Local::now(); - let parent = Parent::new(&index, parent_tree, ignore_ctime, ignore_inode); let file_archiver = FileArchiver::new(be.clone(), index.clone(), indexer.clone(), config)?; let tree_archiver = TreeArchiver::new(be.clone(), index, indexer.clone(), config, summary)?; Ok(Self { file_archiver, tree_archiver, - parent_tree, parent, indexer, be, @@ -62,6 +56,7 @@ impl Archiver { pub fn archive( mut self, + index: &I, src: R, backup_path: &Path, as_path: Option<&PathBuf>, @@ -112,7 +107,7 @@ impl Archiver { scope(|scope| -> RusticResult<_> { // use parent snapshot - iter.filter_map(|item| match self.parent.process(item) { + iter.filter_map(|item| match self.parent.process(index, item) { Ok(item) => Some(item), Err(err) => { warn!("ignoring error reading parent snapshot: {err:?}"); @@ -134,7 +129,7 @@ impl Archiver { .unwrap()?; let stats = self.file_archiver.finalize()?; - let (id, mut summary) = self.tree_archiver.finalize(self.parent_tree)?; + let (id, mut summary) = self.tree_archiver.finalize(self.parent.tree_id())?; stats.apply(&mut summary, BlobType::Data); self.snap.tree = id; diff --git a/crates/rustic_core/src/archiver/parent.rs b/crates/rustic_core/src/archiver/parent.rs index a044e664a..bcf6e0f71 100644 --- a/crates/rustic_core/src/archiver/parent.rs +++ b/crates/rustic_core/src/archiver/parent.rs @@ -10,11 +10,12 @@ use crate::{ id::Id, index::IndexedBackend, RusticResult, }; -pub(crate) struct Parent { +#[derive(Debug)] +pub struct Parent { + tree_id: Option, tree: Option, node_idx: usize, stack: Vec<(Option, usize)>, - be: BE, ignore_ctime: bool, ignore_inode: bool, } @@ -38,8 +39,8 @@ impl ParentResult { pub(crate) type ItemWithParent = TreeType<(O, ParentResult<()>), ParentResult>; -impl Parent { - pub(crate) fn new( +impl Parent { + pub(crate) fn new( be: &BE, tree_id: Option, ignore_ctime: bool, @@ -54,10 +55,10 @@ impl Parent { } }); Self { + tree_id, tree, node_idx: 0, stack: Vec::new(), - be: be.clone(), ignore_ctime, ignore_inode, } @@ -105,24 +106,22 @@ impl Parent { }) } - fn set_dir(&mut self, name: &OsStr) { - let tree = match self.p_node(name) { - Some(p_node) => { - if let Some(tree_id) = p_node.subtree { - match Tree::from_backend(&self.be, tree_id) { - Ok(tree) => Some(tree), - Err(err) => { - warn!("ignoring error when loading parent tree {tree_id}: {err}"); - None - } - } - } else { + fn set_dir(&mut self, be: &BE, name: &OsStr) { + let tree = self.p_node(name).and_then(|p_node| { + p_node.subtree.map_or_else( + || { warn!("ignoring parent node {}: is no tree!", p_node.name); None - } - } - None => None, - }; + }, + |tree_id| match Tree::from_backend(be, tree_id) { + Ok(tree) => Some(tree), + Err(err) => { + warn!("ignoring error when loading parent tree {tree_id}: {err}"); + None + } + }, + ) + }); self.stack.push((self.tree.take(), self.node_idx)); self.tree = tree; self.node_idx = 0; @@ -140,8 +139,13 @@ impl Parent { Ok(()) } - pub(crate) fn process( + pub(crate) fn tree_id(&self) -> Option { + self.tree_id + } + + pub(crate) fn process( &mut self, + be: &BE, item: TreeType, ) -> RusticResult> { let result = match item { @@ -149,7 +153,7 @@ impl Parent { let parent_result = self .is_parent(&node, &tree) .map(|node| node.subtree.unwrap()); - self.set_dir(&tree); + self.set_dir(be, &tree); TreeType::NewTree((path, node, parent_result)) } TreeType::EndTree => { @@ -157,7 +161,7 @@ impl Parent { TreeType::EndTree } TreeType::Other((path, mut node, open)) => { - let be = self.be.clone(); + let be = be.clone(); let parent = self.is_parent(&node, &node.name()); let parent = match parent { ParentResult::Matched(p_node) => { diff --git a/crates/rustic_core/src/backend/ignore.rs b/crates/rustic_core/src/backend/ignore.rs index e0c65198d..3c4eb32de 100644 --- a/crates/rustic_core/src/backend/ignore.rs +++ b/crates/rustic_core/src/backend/ignore.rs @@ -129,7 +129,7 @@ pub struct LocalSourceFilterOptions { impl LocalSource { pub fn new( save_opts: LocalSourceSaveOptions, - filter_opts: LocalSourceFilterOptions, + filter_opts: &LocalSourceFilterOptions, backup_paths: &[impl AsRef], ) -> RusticResult { let mut walk_builder = WalkBuilder::new(&backup_paths[0]); @@ -140,13 +140,13 @@ impl LocalSource { let mut override_builder = OverrideBuilder::new("/"); - for g in filter_opts.glob { + for g in &filter_opts.glob { _ = override_builder - .add(&g) + .add(g) .map_err(IgnoreErrorKind::GenericError)?; } - for file in filter_opts.glob_file { + for file in &filter_opts.glob_file { for line in std::fs::read_to_string(file) .map_err(IgnoreErrorKind::FromIoError)? .lines() @@ -160,13 +160,13 @@ impl LocalSource { _ = override_builder .case_insensitive(true) .map_err(IgnoreErrorKind::GenericError)?; - for g in filter_opts.iglob { + for g in &filter_opts.iglob { _ = override_builder - .add(&g) + .add(g) .map_err(IgnoreErrorKind::GenericError)?; } - for file in filter_opts.iglob_file { + for file in &filter_opts.iglob_file { for line in std::fs::read_to_string(file) .map_err(IgnoreErrorKind::FromIoError)? .lines() @@ -192,10 +192,11 @@ impl LocalSource { .map_err(IgnoreErrorKind::GenericError)?, ); + let exclude_if_present = filter_opts.exclude_if_present.clone(); if !filter_opts.exclude_if_present.is_empty() { _ = walk_builder.filter_entry(move |entry| match entry.file_type() { Some(tpe) if tpe.is_dir() => { - for file in &filter_opts.exclude_if_present { + for file in &exclude_if_present { if entry.path().join(file).exists() { return false; } diff --git a/crates/rustic_core/src/commands.rs b/crates/rustic_core/src/commands.rs index 96442aa8c..630ef7f0d 100644 --- a/crates/rustic_core/src/commands.rs +++ b/crates/rustic_core/src/commands.rs @@ -1,3 +1,4 @@ +pub mod backup; pub mod cat; pub mod check; pub mod config; diff --git a/crates/rustic_core/src/commands/backup.rs b/crates/rustic_core/src/commands/backup.rs new file mode 100644 index 000000000..6e0c884bb --- /dev/null +++ b/crates/rustic_core/src/commands/backup.rs @@ -0,0 +1,221 @@ +//! `backup` subcommand + +/// App-local prelude includes `app_reader()`/`app_writer()`/`app_config()` +/// accessors along with logging macros. Customize as you see fit. +use log::info; + +use std::path::PathBuf; + +use path_dedot::ParseDot; +use serde::Deserialize; + +use crate::{ + archiver::{parent::Parent, Archiver}, + repository::{IndexedIds, IndexedTree}, + DryRunBackend, Id, LocalSource, LocalSourceFilterOptions, LocalSourceSaveOptions, Open, + PathList, ProgressBars, Repository, RusticResult, SnapshotFile, SnapshotGroup, + SnapshotGroupCriterion, StdinSource, +}; + +/// `backup` subcommand +#[cfg_attr(feature = "clap", derive(clap::Parser))] +#[cfg_attr(feature = "merge", derive(merge::Merge))] +#[derive(Clone, Default, Debug, Deserialize)] +#[serde(default, rename_all = "kebab-case", deny_unknown_fields)] +// Note: using sources and source within this struct is a hack to support serde(deny_unknown_fields) +// for deserializing the backup options from TOML +// Unfortunately we cannot work with nested flattened structures, see +// https://github.com/serde-rs/serde/issues/1547 +// A drawback is that a wrongly set "source(s) = ..." won't get correct error handling and need to be manually checked, see below. +#[allow(clippy::struct_excessive_bools)] +pub struct ParentOpts { + /// Group snapshots by any combination of host,label,paths,tags to find a suitable parent (default: host,label,paths) + #[cfg_attr( + feature = "clap", + clap( + long, + short = 'g', + value_name = "CRITERION", + help_heading = "Options for parent processing" + ) + )] + pub group_by: Option, + + /// Snapshot to use as parent + #[cfg_attr( + feature = "clap", + clap( + long, + value_name = "SNAPSHOT", + conflicts_with = "force", + help_heading = "Options for parent processing" + ) + )] + pub parent: Option, + + /// Use no parent, read all files + #[cfg_attr( + feature = "clap", + clap( + long, + short, + conflicts_with = "parent", + help_heading = "Options for parent processing" + ) + )] + #[cfg_attr(feature = "merge", merge(strategy = merge::bool::overwrite_false))] + pub force: bool, + + /// Ignore ctime changes when checking for modified files + #[cfg_attr( + feature = "clap", + clap( + long, + conflicts_with = "force", + help_heading = "Options for parent processing" + ) + )] + #[cfg_attr(feature = "merge", merge(strategy = merge::bool::overwrite_false))] + pub ignore_ctime: bool, + + /// Ignore inode number changes when checking for modified files + #[cfg_attr( + feature = "clap", + clap( + long, + conflicts_with = "force", + help_heading = "Options for parent processing" + ) + )] + #[cfg_attr(feature = "merge", merge(strategy = merge::bool::overwrite_false))] + pub ignore_inode: bool, +} + +impl ParentOpts { + pub fn get_parent( + &self, + repo: &Repository, + snap: &SnapshotFile, + backup_stdin: bool, + ) -> (Option, Parent) { + let parent = match (backup_stdin, self.force, &self.parent) { + (true, _, _) | (false, true, _) => None, + (false, false, None) => { + // get suitable snapshot group from snapshot and opts.group_by. This is used to filter snapshots for the parent detection + let group = SnapshotGroup::from_sn( + snap, + self.group_by + .unwrap_or_else(SnapshotGroupCriterion::default), + ); + SnapshotFile::latest( + repo.dbe(), + |snap| snap.has_group(&group), + &repo.pb.progress_counter(""), + ) + .ok() + } + (false, false, Some(parent)) => SnapshotFile::from_id(repo.dbe(), parent).ok(), + }; + + let (parent_tree, parent_id) = parent.map(|parent| (parent.tree, parent.id)).unzip(); + + ( + parent_id, + Parent::new( + repo.index(), + parent_tree, + self.ignore_ctime, + self.ignore_inode, + ), + ) + } +} + +#[cfg_attr(feature = "clap", derive(clap::Parser))] +#[cfg_attr(feature = "merge", derive(merge::Merge))] +#[derive(Clone, Default, Debug, Deserialize)] +#[serde(default, rename_all = "kebab-case", deny_unknown_fields)] +pub struct BackupOpts { + /// Set filename to be used when backing up from stdin + #[cfg_attr( + feature = "clap", + clap(long, value_name = "FILENAME", default_value = "stdin") + )] + #[cfg_attr(feature = "merge", merge(skip))] + pub stdin_filename: String, + + /// Manually set backup path in snapshot + #[cfg_attr(feature = "clap", clap(long, value_name = "PATH"))] + pub as_path: Option, + + #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] + pub parent_opts: ParentOpts, + + #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] + pub ignore_save_opts: LocalSourceSaveOptions, + + #[cfg_attr(feature = "clap", clap(flatten))] + #[serde(flatten)] + pub ignore_filter_opts: LocalSourceFilterOptions, +} + +pub(crate) fn backup( + repo: &Repository, + opts: &BackupOpts, + source: PathList, + mut snap: SnapshotFile, + dry_run: bool, +) -> RusticResult { + let index = repo.index(); + + let backup_stdin = source == PathList::from_string("-", false)?; + let backup_path = if backup_stdin { + vec![PathBuf::from(&opts.stdin_filename)] + } else { + source.paths() + }; + + let as_path = opts + .as_path + .as_ref() + .map(|p| -> RusticResult<_> { Ok(p.parse_dot()?.to_path_buf()) }) + .transpose()?; + + match &as_path { + Some(p) => snap.paths.set_paths(&[p.clone()])?, + None => snap.paths.set_paths(&backup_path)?, + }; + + let (parent_id, parent) = opts.parent_opts.get_parent(repo, &snap, backup_stdin); + match parent_id { + Some(id) => { + info!("using parent {}", id); + snap.parent = Some(id); + } + None => { + info!("using no parent"); + } + }; + + let be = DryRunBackend::new(repo.dbe().clone(), dry_run); + info!("starting to backup {source}..."); + let archiver = Archiver::new(be, index.clone(), repo.config(), parent, snap)?; + let p = repo.pb.progress_bytes("determining size..."); + + let snap = if backup_stdin { + let path = &backup_path[0]; + let src = StdinSource::new(path.clone())?; + archiver.archive(repo.index(), src, path, as_path.as_ref(), &p)? + } else { + let src = LocalSource::new( + opts.ignore_save_opts, + &opts.ignore_filter_opts, + &backup_path, + )?; + archiver.archive(repo.index(), src, &backup_path[0], as_path.as_ref(), &p)? + }; + + Ok(snap) +} diff --git a/crates/rustic_core/src/commands/cat.rs b/crates/rustic_core/src/commands/cat.rs index bfd9e852d..2af0ad84f 100644 --- a/crates/rustic_core/src/commands/cat.rs +++ b/crates/rustic_core/src/commands/cat.rs @@ -4,7 +4,7 @@ use bytes::Bytes; use crate::{ error::CommandErrorKind, - repository::{Indexed, Open, Repository}, + repository::{IndexedFull, IndexedTree, Open, Repository}, BlobType, DecryptReadBackend, FileType, Id, IndexedBackend, ProgressBars, ReadBackend, RusticResult, SnapshotFile, Tree, }; @@ -19,7 +19,7 @@ pub(crate) fn cat_file( Ok(data) } -pub(crate) fn cat_blob( +pub(crate) fn cat_blob( repo: &Repository, tpe: BlobType, id: &str, @@ -30,7 +30,7 @@ pub(crate) fn cat_blob( Ok(data) } -pub(crate) fn cat_tree( +pub(crate) fn cat_tree( repo: &Repository, snap: &str, sn_filter: impl FnMut(&SnapshotFile) -> bool + Send + Sync, diff --git a/crates/rustic_core/src/commands/dump.rs b/crates/rustic_core/src/commands/dump.rs index fd499b38e..6d7336037 100644 --- a/crates/rustic_core/src/commands/dump.rs +++ b/crates/rustic_core/src/commands/dump.rs @@ -2,11 +2,11 @@ use std::io::Write; use crate::{ error::CommandErrorKind, - repository::{Indexed, Repository}, + repository::{IndexedFull, IndexedTree, Repository}, BlobType, IndexedBackend, Node, NodeType, RusticResult, }; -pub(crate) fn dump( +pub(crate) fn dump( repo: &Repository, node: &Node, w: &mut impl Write, diff --git a/crates/rustic_core/src/commands/restore.rs b/crates/rustic_core/src/commands/restore.rs index 73c51fd11..a8fa1c7f1 100644 --- a/crates/rustic_core/src/commands/restore.rs +++ b/crates/rustic_core/src/commands/restore.rs @@ -17,9 +17,11 @@ use itertools::Itertools; use rayon::ThreadPoolBuilder; use crate::{ - error::CommandErrorKind, hash, repository::Indexed, DecryptReadBackend, FileType, Id, - IndexedBackend, LocalDestination, Node, NodeType, Open, Progress, ProgressBars, ReadBackend, - Repository, RusticResult, + error::CommandErrorKind, + hash, + repository::{IndexedFull, IndexedTree}, + DecryptReadBackend, FileType, Id, IndexedBackend, LocalDestination, Node, NodeType, Open, + Progress, ProgressBars, ReadBackend, Repository, RusticResult, }; pub(crate) mod constants { @@ -65,7 +67,7 @@ pub struct RestoreStats { } impl RestoreOpts { - pub(crate) fn restore( + pub(crate) fn restore( self, file_infos: RestoreInfos, repo: &Repository, @@ -83,7 +85,7 @@ impl RestoreOpts { } /// collect restore information, scan existing files, create needed dirs and remove superfluous files - pub(crate) fn collect_and_prepare( + pub(crate) fn collect_and_prepare( self, repo: &Repository, mut node_streamer: impl Iterator>, diff --git a/crates/rustic_core/src/lib.rs b/crates/rustic_core/src/lib.rs index a70e3b776..94a6bf577 100644 --- a/crates/rustic_core/src/lib.rs +++ b/crates/rustic_core/src/lib.rs @@ -101,7 +101,6 @@ pub(crate) mod repository; // rustic_core Public API pub use crate::{ - archiver::Archiver, backend::{ cache::Cache, decrypt::{DecryptBackend, DecryptFullBackend, DecryptReadBackend, DecryptWriteBackend}, @@ -120,6 +119,7 @@ pub use crate::{ }, chunker::random_poly, commands::{ + backup::BackupOpts, check::CheckOpts, config::ConfigOpts, forget::{ForgetGroup, ForgetGroups, ForgetSnapshot, KeepOptions}, diff --git a/crates/rustic_core/src/repofile/snapshotfile.rs b/crates/rustic_core/src/repofile/snapshotfile.rs index 323cd2168..604c336e1 100644 --- a/crates/rustic_core/src/repofile/snapshotfile.rs +++ b/crates/rustic_core/src/repofile/snapshotfile.rs @@ -471,10 +471,10 @@ impl Ord for SnapshotFile { #[allow(clippy::struct_excessive_bools)] #[derive(DeserializeFromStr, Clone, Debug, Copy)] pub struct SnapshotGroupCriterion { - hostname: bool, - label: bool, - paths: bool, - tags: bool, + pub hostname: bool, + pub label: bool, + pub paths: bool, + pub tags: bool, } impl Default for SnapshotGroupCriterion { diff --git a/crates/rustic_core/src/repository.rs b/crates/rustic_core/src/repository.rs index 374cf77e8..97f07462c 100644 --- a/crates/rustic_core/src/repository.rs +++ b/crates/rustic_core/src/repository.rs @@ -29,6 +29,7 @@ use crate::{ }, commands::{ self, + backup::BackupOpts, check::CheckOpts, config::ConfigOpts, forget::{ForgetGroups, KeepOptions}, @@ -41,7 +42,7 @@ use crate::{ repofile::RepoFile, repofile::{configfile::ConfigFile, keyfile::find_key_in_backend}, BlobType, DecryptFullBackend, Id, IndexBackend, IndexedBackend, LocalDestination, - NoProgressBars, Node, NodeStreamer, ProgressBars, PruneOpts, PrunePlan, RusticResult, + NoProgressBars, Node, NodeStreamer, PathList, ProgressBars, PruneOpts, PrunePlan, RusticResult, SnapshotFile, SnapshotGroup, SnapshotGroupCriterion, Tree, TreeStreamerOptions, }; @@ -537,11 +538,29 @@ impl Repository { opts.get_plan(self) } - pub fn to_indexed(self) -> RusticResult>> { + pub fn to_indexed(self) -> RusticResult>> { let index = IndexBackend::new(self.dbe(), &self.pb.progress_counter(""))?; let status = IndexedStatus { open: self.status, index, + marker: std::marker::PhantomData, + }; + Ok(Repository { + name: self.name, + be: self.be, + be_hot: self.be_hot, + opts: self.opts, + pb: self.pb, + status, + }) + } + + pub fn to_indexed_ids(self) -> RusticResult>> { + let index = IndexBackend::only_full_trees(self.dbe(), &self.pb.progress_counter(""))?; + let status = IndexedStatus { + open: self.status, + index, + marker: std::marker::PhantomData, }; Ok(Repository { name: self.name, @@ -567,12 +586,15 @@ impl Repository { } } -pub trait Indexed: Open { +pub trait IndexedTree: Open { type I: IndexedBackend; fn index(&self) -> &Self::I; } -impl Indexed for Repository { +pub trait IndexedIds: IndexedTree {} +pub trait IndexedFull: IndexedIds {} + +impl IndexedTree for Repository { type I = S::I; fn index(&self) -> &Self::I { self.status.index() @@ -580,12 +602,16 @@ impl Indexed for Repository { } #[derive(Debug)] -pub struct IndexedStatus { +pub struct IndexedStatus { open: S, index: IndexBackend, + marker: std::marker::PhantomData, } -impl Indexed for IndexedStatus { +pub struct IdIndex {} +pub struct FullIndex {} + +impl IndexedTree for IndexedStatus { type I = IndexBackend; fn index(&self) -> &Self::I { @@ -593,7 +619,11 @@ impl Indexed for IndexedStatus { } } -impl Open for IndexedStatus { +impl IndexedIds for IndexedStatus {} +impl IndexedIds for IndexedStatus {} +impl IndexedFull for IndexedStatus {} + +impl Open for IndexedStatus { type DBE = S::DBE; fn key(&self) -> &Key { @@ -610,7 +640,7 @@ impl Open for IndexedStatus { } } -impl Repository { +impl Repository { pub fn node_from_snapshot_path( &self, snap_path: &str, @@ -624,10 +654,6 @@ impl Repository { Tree::node_from_path(self.index(), snap.tree, Path::new(path)) } - pub fn cat_blob(&self, tpe: BlobType, id: &str) -> RusticResult { - commands::cat::cat_blob(self, tpe, id) - } - pub fn cat_tree( &self, snap: &str, @@ -636,10 +662,6 @@ impl Repository { commands::cat::cat_tree(self, snap, sn_filter) } - pub fn dump(&self, node: &Node, w: &mut impl Write) -> RusticResult<()> { - commands::dump::dump(self, node, w) - } - pub fn ls( &self, node: &Node, @@ -649,6 +671,38 @@ impl Repository { NodeStreamer::new_with_glob(self.index().clone(), node, streamer_opts, recursive) } + pub fn restore( + &self, + restore_infos: RestoreInfos, + opts: &RestoreOpts, + node_streamer: impl Iterator>, + dest: &LocalDestination, + ) -> RusticResult<()> { + opts.restore(restore_infos, self, node_streamer, dest) + } +} + +impl Repository { + pub fn backup( + &self, + opts: &BackupOpts, + source: PathList, + snap: SnapshotFile, + dry_run: bool, + ) -> RusticResult { + commands::backup::backup(self, opts, source, snap, dry_run) + } +} + +impl Repository { + pub fn cat_blob(&self, tpe: BlobType, id: &str) -> RusticResult { + commands::cat::cat_blob(self, tpe, id) + } + + pub fn dump(&self, node: &Node, w: &mut impl Write) -> RusticResult<()> { + commands::dump::dump(self, node, w) + } + /// Prepare the restore. /// If `dry_run` is set to false, it will also: /// - remove existing files from the destination, if `opts.delete` is set to true @@ -662,14 +716,4 @@ impl Repository { ) -> RusticResult { opts.collect_and_prepare(self, node_streamer, dest, dry_run) } - - pub fn restore( - &self, - restore_infos: RestoreInfos, - opts: &RestoreOpts, - node_streamer: impl Iterator>, - dest: &LocalDestination, - ) -> RusticResult<()> { - opts.restore(restore_infos, self, node_streamer, dest) - } } diff --git a/src/commands/backup.rs b/src/commands/backup.rs index ef11bfa98..1060f108a 100644 --- a/src/commands/backup.rs +++ b/src/commands/backup.rs @@ -11,25 +11,17 @@ use abscissa_core::{Command, Runnable, Shutdown}; use anyhow::{bail, Result}; use log::{debug, info, warn}; -use std::path::PathBuf; -use std::str::FromStr; - use chrono::Local; use merge::Merge; -use path_dedot::ParseDot; use serde::Deserialize; -use rustic_core::{ - Archiver, DryRunBackend, IndexBackend, LocalSource, LocalSourceFilterOptions, - LocalSourceSaveOptions, Open, PathList, ProgressBars, SnapshotFile, SnapshotGroup, - SnapshotGroupCriterion, SnapshotOptions, StdinSource, -}; +use rustic_core::{BackupOpts, PathList, SnapshotFile, SnapshotOptions}; /// `backup` subcommand #[derive(Clone, Command, Default, Debug, clap::Parser, Deserialize, Merge)] #[serde(default, rename_all = "kebab-case", deny_unknown_fields)] -// Note: using cli_sources, sources and source within this strict is a hack to support serde(deny_unknown_fields) +// Note: using cli_sources, sources and source within this struct is a hack to support serde(deny_unknown_fields) // for deserializing the backup options from TOML // Unfortunately we cannot work with nested flattened structures, see // https://github.com/serde-rs/serde/issues/1547 @@ -43,68 +35,9 @@ pub struct BackupCmd { #[serde(skip)] cli_sources: Vec, - /// Group snapshots by any combination of host,label,paths,tags to find a suitable parent (default: host,label,paths) - #[clap( - long, - short = 'g', - value_name = "CRITERION", - help_heading = "Options for parent processing" - )] - group_by: Option, - - /// Snapshot to use as parent - #[clap( - long, - value_name = "SNAPSHOT", - conflicts_with = "force", - help_heading = "Options for parent processing" - )] - parent: Option, - - /// Use no parent, read all files - #[clap( - long, - short, - conflicts_with = "parent", - help_heading = "Options for parent processing" - )] - #[merge(strategy = merge::bool::overwrite_false)] - force: bool, - - /// Ignore ctime changes when checking for modified files - #[clap( - long, - conflicts_with = "force", - help_heading = "Options for parent processing" - )] - #[merge(strategy = merge::bool::overwrite_false)] - ignore_ctime: bool, - - /// Ignore inode number changes when checking for modified files - #[clap( - long, - conflicts_with = "force", - help_heading = "Options for parent processing" - )] - #[merge(strategy = merge::bool::overwrite_false)] - ignore_inode: bool, - - /// Set filename to be used when backing up from stdin - #[clap(long, value_name = "FILENAME", default_value = "stdin")] - #[merge(skip)] - stdin_filename: String, - - /// Manually set backup path in snapshot - #[clap(long, value_name = "PATH")] - as_path: Option, - - #[clap(flatten)] - #[serde(flatten)] - ignore_save_opts: LocalSourceSaveOptions, - #[clap(flatten)] #[serde(flatten)] - ignore_filter_opts: LocalSourceFilterOptions, + backup_opts: BackupOpts, #[clap(flatten, next_help_heading = "Snapshot options")] #[serde(flatten)] @@ -151,9 +84,8 @@ impl BackupCmd { .join(" "); let config = RUSTIC_APP.config(); - let progress_options = &config.global.progress_options; - let repo = open_repository(&config)?; + let repo = open_repository(&config)?.to_indexed_ids()?; // manually check for a "source" field, check is not done by serde, see above. if !config.backup.source.is_empty() { @@ -195,25 +127,15 @@ impl BackupCmd { } }; - let index = - IndexBackend::only_full_trees(repo.dbe(), &progress_options.progress_counter(""))?; - for source in sources { let mut opts = self.clone(); - let index = index.clone(); - let backup_stdin = source == PathList::from_string("-", false)?; - let backup_path = if backup_stdin { - vec![PathBuf::from(&opts.stdin_filename)] - } else { - source.paths() - }; // merge Options from config file, if given if let Some(idx) = config_sources.iter().position(|s| s == &source) { info!("merging source={source} section from config file"); opts.merge(config_opts[idx].clone()); } - if let Some(path) = &opts.as_path { + if let Some(path) = &opts.backup_opts.as_path { // as_path only works in combination with a single target if source.len() > 1 { bail!("as-path only works with a single target!"); @@ -230,79 +152,13 @@ impl BackupCmd { // merge "backup" section from config file, if given opts.merge(config.backup.clone()); - let be = DryRunBackend::new(repo.dbe().clone(), config.global.dry_run); - info!("starting to backup {source}..."); - let as_path = opts.as_path.map(|p| { - match p.parse_dot() { - Ok(it) => it, - Err(err) => { - status_err!("{}", err); - RUSTIC_APP.shutdown(Shutdown::Crash); - } - } - .to_path_buf() - }); - - let mut snap = SnapshotFile::new_from_options(&opts.snap_opts, time, command.clone())?; - match &as_path { - Some(p) => snap.paths.set_paths(&[p.clone()])?, - None => snap.paths.set_paths(&backup_path)?, - }; - - // get suitable snapshot group from snapshot and opts.group_by. This is used to filter snapshots for the parent detection - let group = SnapshotGroup::from_sn( - &snap, - opts.group_by.unwrap_or_else(|| { - SnapshotGroupCriterion::from_str("host,label,paths").unwrap() - }), - ); - - let parent = match (backup_stdin, opts.force, opts.parent.clone()) { - (true, _, _) | (false, true, _) => None, - (false, false, None) => SnapshotFile::latest( - &be, - |snap| snap.has_group(&group), - &progress_options.progress_counter(""), - ) - .ok(), - (false, false, Some(parent)) => SnapshotFile::from_id(&be, &parent).ok(), - }; - - let parent_tree = match &parent { - Some(parent) => { - info!("using parent {}", parent.id); - snap.parent = Some(parent.id); - Some(parent.tree) - } - None => { - info!("using no parent"); - None - } - }; - - let archiver = Archiver::new( - be, - index, - repo.config(), - parent_tree, - opts.ignore_ctime, - opts.ignore_inode, + let snap = SnapshotFile::new_from_options(&opts.snap_opts, time, command.clone())?; + let snap = repo.backup( + &opts.backup_opts, + source.clone(), snap, + config.global.dry_run, )?; - let p = progress_options.progress_bytes("determining size..."); - - let snap = if backup_stdin { - let path = &backup_path[0]; - let src = StdinSource::new(path.clone())?; - archiver.archive(src, path, as_path.as_ref(), &p)? - } else { - let src = LocalSource::new( - opts.ignore_save_opts, - opts.ignore_filter_opts.clone(), - &backup_path, - )?; - archiver.archive(src, &backup_path[0], as_path.as_ref(), &p)? - }; if opts.json { let mut stdout = std::io::stdout(); diff --git a/src/commands/diff.rs b/src/commands/diff.rs index b4be36096..3f3ea69ce 100644 --- a/src/commands/diff.rs +++ b/src/commands/diff.rs @@ -100,7 +100,7 @@ impl DiffCmd { .is_dir(); let src = LocalSource::new( LocalSourceSaveOptions::default(), - self.ignore_opts.clone(), + &self.ignore_opts, &[&path2], )? .map(|item| {