From 7269708effbaf6c9fbff57493b418614992c1dd7 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Fri, 23 Jun 2023 08:20:29 +0200 Subject: [PATCH] move repoinfo to library --- crates/rustic_core/src/commands.rs | 1 + crates/rustic_core/src/commands/repoinfo.rs | 136 ++++++++++++ crates/rustic_core/src/lib.rs | 7 +- crates/rustic_core/src/repository.rs | 49 ++--- src/commands/repoinfo.rs | 228 +++++++++++++------- src/helpers.rs | 33 +-- 6 files changed, 315 insertions(+), 139 deletions(-) create mode 100644 crates/rustic_core/src/commands/repoinfo.rs diff --git a/crates/rustic_core/src/commands.rs b/crates/rustic_core/src/commands.rs index a91af1f86..212d7b3a0 100644 --- a/crates/rustic_core/src/commands.rs +++ b/crates/rustic_core/src/commands.rs @@ -1,2 +1,3 @@ pub mod cat; pub mod check; +pub mod repoinfo; diff --git a/crates/rustic_core/src/commands/repoinfo.rs b/crates/rustic_core/src/commands/repoinfo.rs new file mode 100644 index 000000000..eda3093fc --- /dev/null +++ b/crates/rustic_core/src/commands/repoinfo.rs @@ -0,0 +1,136 @@ +use crate::{ + index::IndexEntry, + repofile::indexfile::{IndexFile, IndexPack}, + BlobType, BlobTypeMap, DecryptReadBackend, FileType, OpenRepository, Progress, ProgressBars, + ReadBackend, Repository, RusticResult, ALL_FILE_TYPES, +}; + +#[derive(Default, Clone, Debug)] +pub struct IndexInfos { + pub blobs: Vec, + pub blobs_delete: Vec, + pub packs: Vec, + pub packs_delete: Vec, +} + +#[derive(Clone, Copy, Debug)] +pub struct BlobInfo { + pub blob_type: BlobType, + pub count: u64, + pub size: u64, + pub data_size: u64, +} + +impl BlobInfo { + pub fn add(&mut self, ie: IndexEntry) { + self.count += 1; + self.size += u64::from(ie.length); + self.data_size += u64::from(ie.data_length()); + } +} + +#[derive(Clone, Copy, Debug)] +pub struct PackInfo { + pub blob_type: BlobType, + pub count: u64, + pub min_size: Option, + pub max_size: Option, +} + +impl PackInfo { + pub fn add(&mut self, ip: &IndexPack) { + self.count += 1; + let size = u64::from(ip.pack_size()); + self.min_size = self + .min_size + .map_or(Some(size), |min_size| Some(min_size.min(size))); + self.max_size = self + .max_size + .map_or(Some(size), |max_size| Some(max_size.max(size))); + } +} + +pub(crate) fn collect_index_infos( + repo: &OpenRepository

, +) -> RusticResult { + let mut blob_info = BlobTypeMap::<()>::default().map(|blob_type, _| BlobInfo { + blob_type, + count: 0, + size: 0, + data_size: 0, + }); + let mut blob_info_delete = blob_info; + let mut pack_info = BlobTypeMap::<()>::default().map(|blob_type, _| PackInfo { + blob_type, + count: 0, + min_size: None, + max_size: None, + }); + let mut pack_info_delete = pack_info; + + let p = repo.pb.progress_counter("scanning index..."); + for index in repo.dbe.stream_all::(&p)? { + let index = index?.1; + for pack in &index.packs { + let tpe = pack.blob_type(); + pack_info[tpe].add(pack); + + for blob in &pack.blobs { + let ie = IndexEntry::from_index_blob(blob, pack.id); + blob_info[tpe].add(ie); + } + } + + for pack in &index.packs_to_delete { + let tpe = pack.blob_type(); + pack_info_delete[tpe].add(pack); + for blob in &pack.blobs { + let ie = IndexEntry::from_index_blob(blob, pack.id); + blob_info_delete[tpe].add(ie); + } + } + } + p.finish(); + + let info = IndexInfos { + blobs: blob_info.into_values().collect(), + blobs_delete: blob_info_delete.into_values().collect(), + packs: pack_info.into_values().collect(), + packs_delete: pack_info_delete.into_values().collect(), + }; + + Ok(info) +} + +#[derive(Default, Clone, Debug)] +pub struct RepoFileInfos { + pub files: Vec, + pub files_hot: Option>, +} + +#[derive(Clone, Copy, Debug)] +pub struct RepoFileInfo { + pub tpe: FileType, + pub count: u64, + pub size: u64, +} + +pub(crate) fn collect_file_info(be: &impl ReadBackend) -> RusticResult> { + let mut files = Vec::with_capacity(ALL_FILE_TYPES.len()); + for tpe in ALL_FILE_TYPES { + let list = be.list_with_size(tpe)?; + let count = list.len() as u64; + let size = list.iter().map(|f| u64::from(f.1)).sum(); + files.push(RepoFileInfo { tpe, count, size }); + } + Ok(files) +} + +pub fn collect_file_infos(repo: &Repository

) -> RusticResult { + let p = repo.pb.progress_spinner("scanning files..."); + let files = collect_file_info(&repo.be)?; + let files_hot = repo.be_hot.as_ref().map(collect_file_info).transpose()?; + p.finish(); + + Ok(RepoFileInfos { files, files_hot }) +} diff --git a/crates/rustic_core/src/lib.rs b/crates/rustic_core/src/lib.rs index d4f082c70..68f90da30 100644 --- a/crates/rustic_core/src/lib.rs +++ b/crates/rustic_core/src/lib.rs @@ -120,7 +120,10 @@ pub use crate::{ BlobLocation, BlobType, BlobTypeMap, Initialize, Sum, }, chunker::random_poly, - commands::check::CheckOpts, + commands::{ + check::CheckOpts, + repoinfo::{BlobInfo, IndexInfos, PackInfo, RepoFileInfo, RepoFileInfos}, + }, crypto::{aespoly1305::Key, hasher::hash}, error::{RusticError, RusticResult}, file::{AddFileResult, FileInfos, RestoreStats}, @@ -142,5 +145,5 @@ pub use crate::{ }, RepoFile, }, - repository::{parse_command, OpenRepository, RepoInfo, Repository, RepositoryOptions}, + repository::{parse_command, OpenRepository, Repository, RepositoryOptions}, }; diff --git a/crates/rustic_core/src/repository.rs b/crates/rustic_core/src/repository.rs index aab3e2af2..ab400c044 100644 --- a/crates/rustic_core/src/repository.rs +++ b/crates/rustic_core/src/repository.rs @@ -7,7 +7,6 @@ use std::{ }; use bytes::Bytes; -use derive_more::Add; use log::{debug, error, info}; use nom::{ @@ -29,11 +28,14 @@ use crate::{ decrypt::DecryptReadBackend, decrypt::DecryptWriteBackend, hotcold::HotColdBackend, FileType, ReadBackend, }, - commands::{self, check::CheckOpts}, + commands::{ + self, + check::CheckOpts, + repoinfo::{IndexInfos, RepoFileInfos}, + }, crypto::aespoly1305::Key, error::RepositoryErrorKind, - index::IndexEntry, - repofile::{configfile::ConfigFile, indexfile::IndexPack, keyfile::find_key_in_backend}, + repofile::{configfile::ConfigFile, keyfile::find_key_in_backend}, BlobType, IndexBackend, NoProgressBars, ProgressBars, RusticResult, SnapshotFile, }; @@ -41,33 +43,6 @@ pub(super) mod constants { pub(super) const MAX_PASSWORD_RETRIES: usize = 5; } -#[derive(Default, Clone, Copy, Add, Debug)] -pub struct RepoInfo { - pub count: u64, - pub size: u64, - pub data_size: u64, - pub pack_count: u64, - pub total_pack_size: u64, - pub min_pack_size: u64, - pub max_pack_size: u64, -} - -impl RepoInfo { - pub fn add(&mut self, ie: IndexEntry) { - self.count += 1; - self.size += u64::from(ie.length); - self.data_size += u64::from(ie.data_length()); - } - - pub fn add_pack(&mut self, ip: &IndexPack) { - self.pack_count += 1; - let size = u64::from(ip.pack_size()); - self.total_pack_size += size; - self.min_pack_size = self.min_pack_size.min(size); - self.max_pack_size = self.max_pack_size.max(size); - } -} - #[serde_as] #[cfg_attr(feature = "clap", derive(clap::Parser))] #[cfg_attr(feature = "merge", derive(merge::Merge))] @@ -201,7 +176,7 @@ pub struct Repository

{ pub be: HotColdBackend, pub be_hot: Option, opts: RepositoryOptions, - pb: P, + pub(crate) pb: P, } impl Repository { @@ -354,6 +329,12 @@ impl

Repository

{ } } +impl Repository

{ + pub fn infos_files(&self) -> RusticResult { + commands::repoinfo::collect_file_infos(self) + } +} + pub(crate) fn get_key(be: &impl ReadBackend, password: Option) -> RusticResult { for _ in 0..constants::MAX_PASSWORD_RETRIES { match password { @@ -403,6 +384,10 @@ impl OpenRepository

{ let index = IndexBackend::new(&self.dbe, &self.pb.progress_counter(""))?; Ok(IndexedRepository { repo: self, index }) } + + pub fn infos_index(&self) -> RusticResult { + commands::repoinfo::collect_index_infos(self) + } } #[derive(Debug)] diff --git a/src/commands/repoinfo.rs b/src/commands/repoinfo.rs index a204d5c74..62fcfa44f 100644 --- a/src/commands/repoinfo.rs +++ b/src/commands/repoinfo.rs @@ -3,19 +3,14 @@ /// App-local prelude includes `app_reader()`/`app_writer()`/`app_config()` /// accessors along with logging macros. Customize as you see fit. use crate::{ - commands::{get_repository, open_repository}, - helpers::bytes_size_to_string, - status_err, Application, RUSTIC_APP, + commands::get_repository, helpers::bytes_size_to_string, status_err, Application, RUSTIC_APP, }; use abscissa_core::{Command, Runnable, Shutdown}; -use crate::helpers::{print_file_info, table_right_from}; +use crate::helpers::table_right_from; use anyhow::Result; -use rustic_core::{ - BlobType, BlobTypeMap, DecryptReadBackend, IndexEntry, IndexFile, Progress, ProgressBars, - RepoInfo, Sum, -}; +use rustic_core::RepoFileInfo; /// `repoinfo` subcommand #[derive(clap::Parser, Command, Debug)] @@ -33,83 +28,55 @@ impl Runnable for RepoInfoCmd { impl RepoInfoCmd { fn inner_run(&self) -> Result<()> { let config = RUSTIC_APP.config(); - let repo = open_repository(get_repository(&config)); - - print_file_info("repository files", &repo.be)?; - - if let Some(hot_be) = &repo.be_hot { - print_file_info("hot repository files", hot_be)?; + let repo = get_repository(&config); + let file_info = repo.infos_files()?; + print_file_info("repository files", file_info.files); + if let Some(info) = file_info.files_hot { + print_file_info("hot repository files", info); } - let mut info = BlobTypeMap::::default(); - info[BlobType::Tree].min_pack_size = u64::MAX; - info[BlobType::Data].min_pack_size = u64::MAX; - let mut info_delete = BlobTypeMap::::default(); - - let p = config - .global - .progress_options - .progress_counter("scanning index..."); - repo.dbe - .stream_all::(&p)? - .into_iter() - .for_each(|index| { - let index = match index { - Ok(it) => it, - Err(err) => { - status_err!("{}", err); - RUSTIC_APP.shutdown(Shutdown::Crash); - } - } - .1; - for pack in &index.packs { - info[pack.blob_type()].add_pack(pack); - - for blob in &pack.blobs { - let ie = IndexEntry::from_index_blob(blob, pack.id); - info[pack.blob_type()].add(ie); - } - } - - for pack in &index.packs_to_delete { - for blob in &pack.blobs { - let ie = IndexEntry::from_index_blob(blob, pack.id); - info_delete[pack.blob_type()].add(ie); - } - } - }); - p.finish(); + let repo = repo.open()?; + let index_info = repo.infos_index()?; let mut table = table_right_from( 1, ["Blob type", "Count", "Total Size", "Total Size in Packs"], ); - for (blob_type, info) in &info { + let mut total_count = 0; + let mut total_data_size = 0; + let mut total_size = 0; + + for blobs in &index_info.blobs { _ = table.add_row([ - format!("{blob_type:?}"), - info.count.to_string(), - bytes_size_to_string(info.data_size), - bytes_size_to_string(info.size), + format!("{:?}", blobs.blob_type), + blobs.count.to_string(), + bytes_size_to_string(blobs.data_size), + bytes_size_to_string(blobs.size), ]); + total_count += blobs.count; + total_data_size += blobs.data_size; + total_size += blobs.size; } - - for (blob_type, info_delete) in &info_delete { - if info_delete.count > 0 { + for blobs in &index_info.blobs_delete { + if blobs.count > 0 { _ = table.add_row([ - format!("{blob_type:?} to delete"), - info_delete.count.to_string(), - bytes_size_to_string(info_delete.data_size), - bytes_size_to_string(info_delete.size), + format!("{:?} to delete", blobs.blob_type), + blobs.count.to_string(), + bytes_size_to_string(blobs.data_size), + bytes_size_to_string(blobs.size), ]); + total_count += blobs.count; + total_data_size += blobs.data_size; + total_size += blobs.size; } } - let total = info.sum() + info_delete.sum(); + _ = table.add_row([ "Total".to_string(), - total.count.to_string(), - bytes_size_to_string(total.data_size), - bytes_size_to_string(total.size), + total_count.to_string(), + bytes_size_to_string(total_data_size), + bytes_size_to_string(total_size), ]); println!(); @@ -120,17 +87,132 @@ impl RepoInfoCmd { ["Blob type", "Pack Count", "Minimum Size", "Maximum Size"], ); - for (blob_type, info) in info { + for packs in index_info.packs { _ = table.add_row([ - format!("{blob_type:?} packs"), - info.pack_count.to_string(), - bytes_size_to_string(info.min_pack_size), - bytes_size_to_string(info.max_pack_size), + format!("{:?} packs", packs.blob_type), + packs.count.to_string(), + packs + .min_size + .map_or("-".to_string(), |s| bytes_size_to_string(s)), + packs + .max_size + .map_or("-".to_string(), |s| bytes_size_to_string(s)), ]); } + for packs in index_info.packs_delete { + if packs.count > 0 { + _ = table.add_row([ + format!("{:?} packs to delete", packs.blob_type), + packs.count.to_string(), + packs + .min_size + .map_or("-".to_string(), |s| bytes_size_to_string(s)), + packs + .max_size + .map_or("-".to_string(), |s| bytes_size_to_string(s)), + ]); + } + } println!(); println!("{table}"); Ok(()) } } + +pub fn print_file_info(text: &str, info: Vec) { + let mut table = table_right_from(1, ["File type", "Count", "Total Size"]); + let mut total_count = 0; + let mut total_size = 0; + for row in info { + _ = table.add_row([ + format!("{:?}", row.tpe), + row.count.to_string(), + bytes_size_to_string(row.size), + ]); + total_count += row.count; + total_size += row.size; + } + println!("{text}"); + _ = table.add_row([ + "Total".to_string(), + total_count.to_string(), + bytes_size_to_string(total_size), + ]); + + println!(); + println!("{table}"); + println!(); +} + +pub fn print_index_info(index_info: IndexInfos) { + let mut table = table_right_from( + 1, + ["Blob type", "Count", "Total Size", "Total Size in Packs"], + ); + + let mut total_count = 0; + let mut total_data_size = 0; + let mut total_size = 0; + + for blobs in &index_info.blobs { + _ = table.add_row([ + format!("{:?}", blobs.blob_type), + blobs.count.to_string(), + bytes_size_to_string(blobs.data_size), + bytes_size_to_string(blobs.size), + ]); + total_count += blobs.count; + total_data_size += blobs.data_size; + total_size += blobs.size; + } + for blobs in &index_info.blobs_delete { + if blobs.count > 0 { + _ = table.add_row([ + format!("{:?} to delete", blobs.blob_type), + blobs.count.to_string(), + bytes_size_to_string(blobs.data_size), + bytes_size_to_string(blobs.size), + ]); + total_count += blobs.count; + total_data_size += blobs.data_size; + total_size += blobs.size; + } + } + + _ = table.add_row([ + "Total".to_string(), + total_count.to_string(), + bytes_size_to_string(total_data_size), + bytes_size_to_string(total_size), + ]); + + println!(); + println!("{table}"); + + let mut table = table_right_from( + 1, + ["Blob type", "Pack Count", "Minimum Size", "Maximum Size"], + ); + + for packs in index_info.packs { + _ = table.add_row([ + format!("{:?} packs", packs.blob_type), + packs.count.to_string(), + packs.min_size.map_or("-".to_string(), bytes_size_to_string), + packs.max_size.map_or("-".to_string(), bytes_size_to_string), + ]); + } + for packs in index_info.packs_delete { + if packs.count > 0 { + _ = table.add_row([ + format!("{:?} packs to delete", packs.blob_type), + packs.count.to_string(), + packs.min_size.map_or("-".to_string(), bytes_size_to_string), + packs.max_size.map_or("-".to_string(), bytes_size_to_string), + ]); + } + } + println!(); + println!("{table}"); +} diff --git a/src/helpers.rs b/src/helpers.rs index 8101a2ce4..82676fe61 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -16,7 +16,7 @@ use rayon::{ use rustic_core::{ parse_command, BlobType, DecryptWriteBackend, FileType, Id, IndexBackend, IndexedBackend, Indexer, NodeType, OpenRepository, Packer, Progress, ProgressBars, ReadBackend, ReadIndex, - RusticResult, SnapshotFile, TreeStreamerOnce, ALL_FILE_TYPES, + SnapshotFile, TreeStreamerOnce, }; use crate::{application::RUSTIC_APP, config::progress_options::ProgressOptions}; @@ -274,37 +274,6 @@ pub fn table_right_from, T: ToString>(start: usize, ti table } -pub fn print_file_info(text: &str, be: &impl ReadBackend) -> RusticResult<()> { - info!("scanning files..."); - - let mut table = table_right_from(1, ["File type", "Count", "Total Size"]); - let mut total_count = 0; - let mut total_size = 0; - for tpe in ALL_FILE_TYPES { - let list = be.list_with_size(tpe)?; - let count = list.len(); - let size = list.iter().map(|f| u64::from(f.1)).sum(); - _ = table.add_row([ - format!("{tpe:?}"), - count.to_string(), - bytes_size_to_string(size), - ]); - total_count += count; - total_size += size; - } - println!("{text}"); - _ = table.add_row([ - "Total".to_string(), - total_count.to_string(), - bytes_size_to_string(total_size), - ]); - - println!(); - println!("{table}"); - println!(); - Ok(()) -} - #[must_use] pub fn bytes_size_to_string(b: u64) -> String { ByteSize(b).to_string_as(true)