Improve sync logic for deleted folders. (#479)

* Improve sync logic for deleted folders.

* Bump probly-search dependency.

* Move features to account_extras crate.

Closes #466.

* Update lock file.

* Bump patch version.

Author: tmpfs

Cargo.lock

@@ -1,6 +1,7 @@
 [workspace]
 resolver = "2"
 members = [
+  "crates/account_extras",
   "crates/artifact",
   "crates/cli_helpers", 
   "crates/keychain_parser",
@@ -11,7 +12,7 @@ members = [
   "crates/server",
   "crates/sos",
   "crates/test_utils",
-  "crates/vfs",
+  "crates/vfs" 
 ]
 
 [workspace.dependencies]

Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "sos-account-extras"
+version = "0.14.5"
+edition = "2021"
+description = "Extra features for Save Our Secrets local accounts."
+homepage = "https://saveoursecrets.com"
+license = "MIT OR Apache-2.0"
+repository = "https://github.com/saveoursecrets/sdk"
+
+[features]
+preferences = []
+security-report = []
+system-messages = []
+
+[dependencies]
+thiserror.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+serde_with.workspace = true
+tokio.workspace = true
+once_cell.workspace = true
+
+[dependencies.sos-sdk]
+version = "0.14.5"
+path = "../sdk"
+
+[build-dependencies]
+rustc_version = "0.4.0"

crates/account_extras/Cargo.toml

@@ -0,0 +1,12 @@
+use rustc_version::{version_meta, Channel};
+
+fn main() {
+    // Set cfg flags depending on release channel
+    let channel = match version_meta().unwrap().channel {
+        Channel::Stable => "CHANNEL_STABLE",
+        Channel::Beta => "CHANNEL_BETA",
+        Channel::Nightly => "CHANNEL_NIGHTLY",
+        Channel::Dev => "CHANNEL_DEV",
+    };
+    println!("cargo:rustc-cfg={}", channel)
+}

crates/account_extras/build.rs

@@ -0,0 +1,42 @@
+use thiserror::Error;
+
+/// Error type for the account extras library.
+#[derive(Debug, Error)]
+pub enum Error {
+    /// Error generated when a preference is not a number.
+    #[cfg(feature = "preferences")]
+    #[error("expecting integer preference for '{0}'")]
+    PreferenceTypeNumber(String),
+
+    /// Error generated when a preference is not a boolean.
+    #[cfg(feature = "preferences")]
+    #[error("expecting boolean preference for '{0}'")]
+    PreferenceTypeBool(String),
+
+    /// Error generated when a preference is not a string.
+    #[cfg(feature = "preferences")]
+    #[error("expecting string preference for '{0}'")]
+    PreferenceTypeString(String),
+
+    /// Error generated when a preference is not a string list.
+    #[cfg(feature = "preferences")]
+    #[error("expecting string list preference for '{0}'")]
+    PreferenceTypeStringList(String),
+
+    /// Error generated when a system message is not found.
+    #[cfg(feature = "system-messages")]
+    #[error("no system message for key '{0}'")]
+    NoSysMessage(String),
+
+    /// Errors generated by the IO module.
+    #[error(transparent)]
+    Io(#[from] std::io::Error),
+
+    /// Errors generated by the JSON library.
+    #[error(transparent)]
+    Json(#[from] serde_json::Error),
+
+    /// Errors generated by the SDK library.
+    #[error(transparent)]
+    Sdk(#[from] sos_sdk::Error),
+}

crates/account_extras/src/error.rs

@@ -0,0 +1,21 @@
+//! Extra functions for local accounts in the [SDK](https://docs.rs/sos-sdk) for [Save Our Secrets](https://saveoursecrets.com).
+#![deny(missing_docs)]
+#![forbid(unsafe_code)]
+#![cfg_attr(all(doc, CHANNEL_NIGHTLY), feature(doc_auto_cfg))]
+
+mod error;
+
+/// Errors generated by the extras library.
+pub use error::Error;
+
+/// Result type for the extras library.
+pub type Result<T> = std::result::Result<T, Error>;
+
+#[cfg(feature = "preferences")]
+pub mod preferences;
+
+#[cfg(feature = "security-report")]
+pub mod security_report;
+
+#[cfg(feature = "system-messages")]
+pub mod system_messages;

crates/account_extras/src/lib.rs

@@ -4,15 +4,33 @@
 //! of named keys to typed data similar to
 //! the shared preferences provided by an operating
 //! system library.
-use crate::{
-    identity::PublicIdentity, signer::ecdsa::Address, vfs, Error, Paths,
-    Result,
-};
+use crate::{Error, Result};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
+use sos_sdk::{
+    constants::JSON_EXT, identity::PublicIdentity, signer::ecdsa::Address,
+    vfs, Paths,
+};
 use std::{collections::HashMap, fmt, path::PathBuf, sync::Arc};
 use tokio::sync::Mutex;
 
+/// File thats stores account-level preferences.
+pub const PREFERENCES_FILE: &str = "preferences";
+
+/// Path to the file used to store account-level preferences.
+///
+/// # Panics
+///
+/// If this set of paths are global (no user identifier).
+pub fn preferences_path(paths: &Paths) -> PathBuf {
+    if paths.is_global() {
+        panic!("preferences are not accessible for global paths");
+    }
+    let mut vault_path = paths.user_dir().join(PREFERENCES_FILE);
+    vault_path.set_extension(JSON_EXT);
+    vault_path
+}
+
 static CACHE: Lazy<Mutex<HashMap<Address, Arc<Mutex<Preferences>>>>> =
     Lazy::new(|| Mutex::new(HashMap::new()));
 
@@ -55,7 +73,7 @@ impl CachedPreferences {
 
         let mut cache = CACHE.lock().await;
         let paths = Paths::new(&data_dir, address.to_string());
-        let file = paths.preferences();
+        let file = preferences_path(&paths);
         let prefs = if vfs::try_exists(&file).await? {
             let mut prefs = Preferences::new(&paths);
             prefs.load().await?;
@@ -152,7 +170,7 @@ impl Preferences {
     ///
     pub fn new(paths: &Paths) -> Self {
         Self {
-            path: paths.preferences(),
+            path: preferences_path(paths),
             values: Default::default(),
         }
     }

crates/sdk/src/account/preferences.rs renamed to crates/account_extras/src/preferences.rs

@@ -1,13 +1,100 @@
 //! Types for security report generation.
-use crate::{
+use serde::{Deserialize, Serialize};
+use sos_sdk::{
+    account::Account,
+    hex,
     vault::{
         secret::{Secret, SecretId, SecretType},
         Gatekeeper, Summary, VaultId,
     },
     zxcvbn::{Entropy, Score},
     Result,
 };
-use serde::{Deserialize, Serialize};
+
+/// Generate a security report.
+pub async fn generate_security_report<A, E, T, D, R>(
+    account: &A,
+    options: SecurityReportOptions<T, D, R>,
+) -> std::result::Result<SecurityReport<T>, E>
+where
+    A: Account,
+    D: Fn(Vec<String>) -> R + Send + Sync,
+    R: std::future::Future<Output = Vec<T>> + Send + Sync,
+    E: From<A::Error> + From<sos_sdk::Error>,
+{
+    let mut records = Vec::new();
+    let mut hashes = Vec::new();
+    let folders = account.list_folders().await?;
+    let targets: Vec<Summary> = folders
+        .into_iter()
+        .filter(|folder| {
+            if let Some(target) = &options.target {
+                return folder.id() == &target.0;
+            }
+            !options.excludes.contains(folder.id())
+        })
+        .collect();
+
+    for target in targets {
+        let storage = account.storage().await?;
+        let reader = storage.read().await;
+
+        let folder = reader.cache().get(target.id()).unwrap();
+        let keeper = folder.keeper();
+
+        let vault = keeper.vault();
+        let mut password_hashes: Vec<(
+            SecretId,
+            (Option<Entropy>, Vec<u8>),
+            Option<SecretId>,
+        )> = Vec::new();
+
+        if let Some(target) = &options.target {
+            secret_security_report(
+                &target.1,
+                keeper,
+                &mut password_hashes,
+                target.2.as_ref(),
+            )
+            .await?;
+        } else {
+            for secret_id in vault.keys() {
+                secret_security_report(
+                    secret_id,
+                    keeper,
+                    &mut password_hashes,
+                    None,
+                )
+                .await?;
+            }
+        }
+
+        for (secret_id, check, field_id) in password_hashes {
+            let (entropy, sha1) = check;
+
+            let record = SecurityReportRecord {
+                folder: target.clone(),
+                secret_id,
+                field_id,
+                entropy,
+            };
+
+            hashes.push(hex::encode(sha1));
+            records.push(record);
+        }
+    }
+
+    let database_checks =
+        if let Some(database_handler) = options.database_handler {
+            (database_handler)(hashes).await
+        } else {
+            vec![]
+        };
+    Ok(SecurityReport {
+        records,
+        database_checks,
+    })
+}
 
 /// Specific target for a security report.
 pub struct SecurityReportTarget(

crates/sdk/src/account/security_report.rs renamed to crates/account_extras/src/security_report.rs

@@ -13,13 +13,31 @@
 //! changes to the underlying collection. This allows
 //! an interface to show the number of unread system
 //! messages.
-use crate::{vfs, Error, Paths, Result};
+use crate::{Error, Result};
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
+use sos_sdk::{
+    constants::JSON_EXT, time::OffsetDateTime, urn::Urn, vfs, Paths,
+};
 use std::{cmp::Ordering, collections::HashMap, path::PathBuf};
-use time::OffsetDateTime;
 use tokio::sync::broadcast;
-use urn::Urn;
+
+/// File thats stores account-level system messages.
+pub const SYSTEM_MESSAGES_FILE: &str = "system-messages";
+
+/// Path to the file used to store account-level system messages.
+///
+/// # Panics
+///
+/// If this set of paths are global (no user identifier).
+pub fn system_messages_path(paths: &Paths) -> PathBuf {
+    if paths.is_global() {
+        panic!("system messages are not accessible for global paths");
+    }
+    let mut vault_path = paths.user_dir().join(SYSTEM_MESSAGES_FILE);
+    vault_path.set_extension(JSON_EXT);
+    vault_path
+}
 
 /// System messages count.
 #[derive(Debug, Default, Clone, Eq, PartialEq)]
@@ -161,7 +179,7 @@ impl SystemMessages {
     ///
     pub fn new(paths: &Paths) -> Self {
         Self {
-            path: paths.system_messages(),
+            path: system_messages_path(paths),
             messages: Default::default(),
             channel: stream_channel(),
         }

crates/sdk/src/account/system_messages.rs renamed to crates/account_extras/src/system_messages.rs

@@ -1,7 +1,10 @@
 use crate::test_utils::{setup, teardown};
 use anyhow::Result;
 use secrecy::SecretString;
-use sos_net::sdk::{prelude::*, zxcvbn::Score};
+use sos_net::{
+    extras::security_report::*,
+    sdk::{prelude::*, zxcvbn::Score},
+};
 
 #[tokio::test]
 async fn local_security_report() -> Result<()> {
@@ -36,9 +39,14 @@ async fn local_security_report() -> Result<()> {
         target: None,
     };
 
-    let report = account
-        .generate_security_report::<bool, _, _>(report_options)
-        .await?;
+    let report = generate_security_report::<
+        LocalAccount,
+        sos_net::sdk::Error,
+        bool,
+        _,
+        _,
+    >(&account, report_options)
+    .await?;
 
     let weak_record = report
         .records