diff --git a/crates/rustic_core/src/commands/prune.rs b/crates/rustic_core/src/commands/prune.rs index dd4e0c18b..21f0ce124 100644 --- a/crates/rustic_core/src/commands/prune.rs +++ b/crates/rustic_core/src/commands/prune.rs @@ -715,6 +715,8 @@ impl PrunePlan { repo: &OpenRepository

, opts: &PruneOpts, ) -> RusticResult<()> { + repo.warm_up_wait(self.repack_packs().into_iter())?; + let be = &repo.dbe; let pb = &repo.pb; diff --git a/crates/rustic_core/src/error.rs b/crates/rustic_core/src/error.rs index 713abeb6a..38dea4a97 100644 --- a/crates/rustic_core/src/error.rs +++ b/crates/rustic_core/src/error.rs @@ -235,6 +235,8 @@ pub enum RepositoryErrorKind { AccessToConfigFileFailed, /// {0:?} FromNomError(nom::Err<()>), + /// {0:?} + FromThreadPoolbilderError(rayon::ThreadPoolBuildError), /// reading Password failed: `{0:?}` ReadingPasswordFromReaderFailed(std::io::Error), /// reading Password from prompt failed: `{0:?}` diff --git a/crates/rustic_core/src/lib.rs b/crates/rustic_core/src/lib.rs index 601b5344b..0a9213086 100644 --- a/crates/rustic_core/src/lib.rs +++ b/crates/rustic_core/src/lib.rs @@ -146,5 +146,5 @@ pub use crate::{ }, RepoFile, }, - repository::{parse_command, OpenRepository, Repository, RepositoryOptions}, + repository::{OpenRepository, Repository, RepositoryOptions}, }; diff --git a/crates/rustic_core/src/repository.rs b/crates/rustic_core/src/repository.rs index 7f530d05e..fc6ee8a20 100644 --- a/crates/rustic_core/src/repository.rs +++ b/crates/rustic_core/src/repository.rs @@ -36,7 +36,7 @@ use crate::{ crypto::aespoly1305::Key, error::RepositoryErrorKind, repofile::{configfile::ConfigFile, keyfile::find_key_in_backend}, - BlobType, IndexBackend, NoProgressBars, ProgressBars, PruneOpts, PrunePlan, RusticResult, + BlobType, Id, IndexBackend, NoProgressBars, ProgressBars, PruneOpts, PrunePlan, RusticResult, SnapshotFile, SnapshotGroup, SnapshotGroupCriterion, }; @@ -44,6 +44,9 @@ pub(super) mod constants { pub(super) const MAX_PASSWORD_RETRIES: usize = 5; } +mod warm_up; +use warm_up::{warm_up, warm_up_wait}; + #[serde_as] #[cfg_attr(feature = "clap", derive(clap::Parser))] #[cfg_attr(feature = "merge", derive(merge::Merge))] @@ -402,6 +405,14 @@ impl OpenRepository

{ pub fn infos_index(&self) -> RusticResult { commands::repoinfo::collect_index_infos(self) } + + pub fn warm_up(&self, packs: impl ExactSizeIterator) -> RusticResult<()> { + warm_up(self, packs) + } + + pub fn warm_up_wait(&self, packs: impl ExactSizeIterator) -> RusticResult<()> { + warm_up_wait(self, packs) + } } #[derive(Debug)] diff --git a/crates/rustic_core/src/repository/warm_up.rs b/crates/rustic_core/src/repository/warm_up.rs new file mode 100644 index 000000000..433005cd0 --- /dev/null +++ b/crates/rustic_core/src/repository/warm_up.rs @@ -0,0 +1,93 @@ +use std::process::Command; +use std::thread::sleep; + +use log::{debug, warn}; +use rayon::ThreadPoolBuilder; + +use super::parse_command; +use crate::{ + error::RepositoryErrorKind, FileType, Id, OpenRepository, Progress, ProgressBars, ReadBackend, + RusticResult, +}; + +pub(super) mod constants { + pub(super) const MAX_READER_THREADS_NUM: usize = 20; +} + +pub(crate) fn warm_up_wait( + repo: &OpenRepository

, + packs: impl ExactSizeIterator, +) -> RusticResult<()> { + warm_up(repo, packs)?; + if let Some(wait) = repo.opts.warm_up_wait { + let p = repo.pb.progress_spinner(format!("waiting {wait}...")); + sleep(*wait); + p.finish(); + } + Ok(()) +} + +pub(crate) fn warm_up( + repo: &OpenRepository

, + packs: impl ExactSizeIterator, +) -> RusticResult<()> { + if let Some(command) = &repo.opts.warm_up_command { + warm_up_command(packs, command, &repo.pb)?; + } else if repo.opts.warm_up { + warm_up_access(repo, packs)?; + } + Ok(()) +} + +fn warm_up_command( + packs: impl ExactSizeIterator, + command: &str, + pb: &P, +) -> RusticResult<()> { + let p = pb.progress_counter("warming up packs..."); + p.set_length(packs.len() as u64); + for pack in packs { + let actual_command = command.replace("%id", &pack.to_hex()); + debug!("calling {actual_command}..."); + let commands = parse_command::<()>(&actual_command) + .map_err(RepositoryErrorKind::FromNomError)? + .1; + let status = Command::new(commands[0]).args(&commands[1..]).status()?; + if !status.success() { + warn!("warm-up command was not successful for pack {pack:?}. {status}"); + } + } + p.finish(); + Ok(()) +} + +fn warm_up_access( + repo: &OpenRepository

, + packs: impl ExactSizeIterator, +) -> RusticResult<()> { + let mut be = repo.be.clone(); + be.set_option("retry", "false")?; + + let p = repo.pb.progress_counter("warming up packs..."); + p.set_length(packs.len() as u64); + + let pool = ThreadPoolBuilder::new() + .num_threads(constants::MAX_READER_THREADS_NUM) + .build() + .map_err(RepositoryErrorKind::FromThreadPoolbilderError)?; + let p = &p; + let be = &be; + pool.in_place_scope(|s| { + for pack in packs { + s.spawn(move |_| { + // ignore errors as they are expected from the warm-up + _ = be.read_partial(FileType::Pack, &pack, false, 0, 1); + p.inc(1); + }); + } + }); + + p.finish(); + + Ok(()) +} diff --git a/src/commands/prune.rs b/src/commands/prune.rs index dc0e5ef73..c58a6592c 100644 --- a/src/commands/prune.rs +++ b/src/commands/prune.rs @@ -12,8 +12,6 @@ use log::debug; use anyhow::Result; -use crate::helpers::warm_up_wait; - use rustic_core::{PruneOpts, PruneStats, Sum}; /// `prune` subcommand @@ -37,23 +35,15 @@ impl Runnable for PruneCmd { impl PruneCmd { fn inner_run(&self) -> Result<()> { let config = RUSTIC_APP.config(); - let progress_options = &config.global.progress_options; - let repo = open_repository(get_repository(&config)); let pruner = repo.prune_plan(&self.opts)?; print_stats(&pruner.stats); - let dry_run = config.global.dry_run; - warm_up_wait( - &repo, - pruner.repack_packs().into_iter(), - !dry_run, - progress_options, - )?; - - if !dry_run { + if config.global.dry_run { + repo.warm_up(pruner.repack_packs().into_iter())?; + } else { pruner.do_prune(&repo, &self.opts)?; } diff --git a/src/commands/repair.rs b/src/commands/repair.rs index 30604ac4f..5ebfa85e5 100644 --- a/src/commands/repair.rs +++ b/src/commands/repair.rs @@ -19,8 +19,6 @@ use rustic_core::{ ProgressBars, ReadBackend, ReadIndex, SnapshotFile, StringList, Tree, WriteBackend, }; -use crate::helpers::warm_up_wait; - /// `repair` subcommand #[derive(clap::Parser, Command, Debug)] pub(crate) struct RepairCmd { @@ -185,12 +183,7 @@ impl IndexSubCmd { // process packs which are listed but not contained in the index pack_read_header.extend(packs.into_iter().map(|(id, size)| (id, false, None, size))); - warm_up_wait( - &repo, - pack_read_header.iter().map(|(id, _, _, _)| *id), - true, - progress_options, - )?; + repo.warm_up_wait(pack_read_header.iter().map(|(id, _, _, _)| *id))?; let indexer = Indexer::new(be.clone()).into_shared(); let p = progress_options.progress_counter("reading pack headers"); diff --git a/src/commands/restore.rs b/src/commands/restore.rs index e93ed4364..568771b86 100644 --- a/src/commands/restore.rs +++ b/src/commands/restore.rs @@ -33,7 +33,7 @@ use rustic_core::{ TreeStreamerOptions, }; -use crate::{filtering::SnapshotFilter, helpers::warm_up_wait}; +use crate::filtering::SnapshotFilter; pub(crate) mod constants { pub(crate) const MAX_READER_THREADS_NUM: usize = 20; @@ -134,16 +134,11 @@ impl RestoreCmd { if file_infos.restore_size == 0 { info!("all file contents are fine."); + } else if config.global.dry_run { + repo.warm_up(file_infos.to_packs().into_iter())?; } else { - warm_up_wait( - &repo, - file_infos.to_packs().into_iter(), - !config.global.dry_run, - progress_options, - )?; - if !config.global.dry_run { - restore_contents(be, &dest, file_infos)?; - } + repo.warm_up_wait(file_infos.to_packs().into_iter())?; + restore_contents(be, &dest, file_infos)?; } if !config.global.dry_run { diff --git a/src/helpers.rs b/src/helpers.rs index 82676fe61..8375cb6e1 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeSet, process::Command}; +use std::collections::BTreeSet; use abscissa_core::Application; use anyhow::Result; @@ -7,95 +7,15 @@ use comfy_table::{ presets::ASCII_MARKDOWN, Attribute, Cell, CellAlignment, ContentArrangement, Table, }; -use log::{debug, info, trace, warn}; -use rayon::{ - prelude::{IntoParallelRefIterator, ParallelBridge, ParallelIterator}, - ThreadPoolBuilder, -}; +use log::{info, trace}; +use rayon::prelude::{IntoParallelRefIterator, ParallelBridge, ParallelIterator}; use rustic_core::{ - parse_command, BlobType, DecryptWriteBackend, FileType, Id, IndexBackend, IndexedBackend, - Indexer, NodeType, OpenRepository, Packer, Progress, ProgressBars, ReadBackend, ReadIndex, - SnapshotFile, TreeStreamerOnce, + BlobType, DecryptWriteBackend, IndexBackend, IndexedBackend, Indexer, NodeType, OpenRepository, + Packer, Progress, ProgressBars, ReadIndex, SnapshotFile, TreeStreamerOnce, }; -use crate::{application::RUSTIC_APP, config::progress_options::ProgressOptions}; - -pub(super) mod constants { - pub(super) const MAX_READER_THREADS_NUM: usize = 20; -} - -pub(crate) fn warm_up_wait

( - repo: &OpenRepository

, - packs: impl ExactSizeIterator, - wait: bool, - progress_options: &ProgressOptions, -) -> Result<()> { - if let Some(command) = &repo.opts.warm_up_command { - warm_up_command(packs, command, progress_options)?; - } else if repo.opts.warm_up { - warm_up(&repo.be, packs, progress_options)?; - } - if wait { - if let Some(wait) = repo.opts.warm_up_wait { - let p = progress_options.progress_spinner(format!("waiting {wait}...")); - std::thread::sleep(*wait); - p.finish(); - } - } - Ok(()) -} - -pub(crate) fn warm_up_command( - packs: impl ExactSizeIterator, - command: &str, - progress_options: &ProgressOptions, -) -> Result<()> { - let p = progress_options.progress_counter("warming up packs..."); - p.set_length(packs.len() as u64); - for pack in packs { - let actual_command = command.replace("%id", &pack.to_hex()); - debug!("calling {actual_command}..."); - let commands = parse_command::<()>(&actual_command)?.1; - let status = Command::new(commands[0]).args(&commands[1..]).status()?; - if !status.success() { - warn!("warm-up command was not successful for pack {pack:?}. {status}"); - } - } - p.finish(); - Ok(()) -} - -pub(crate) fn warm_up( - be: &impl ReadBackend, - packs: impl ExactSizeIterator, - progress_options: &ProgressOptions, -) -> Result<()> { - let mut be = be.clone(); - be.set_option("retry", "false")?; - - let p = progress_options.progress_counter("warming up packs..."); - p.set_length(packs.len() as u64); - - let pool = ThreadPoolBuilder::new() - .num_threads(constants::MAX_READER_THREADS_NUM) - .build()?; - let p = &p; - let be = &be; - pool.in_place_scope(|s| { - for pack in packs { - s.spawn(move |_| { - // ignore errors as they are expected from the warm-up - _ = be.read_partial(FileType::Pack, &pack, false, 0, 1); - p.inc(1); - }); - } - }); - - p.finish(); - - Ok(()) -} +use crate::application::RUSTIC_APP; pub(crate) fn copy

( snapshots: &[SnapshotFile],