Count megabytes of data

jantb · Jan 30, 2023 · 619a459 · 619a459
1 parent 0e438ad
commit 619a459
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 5 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -21,6 +21,7 @@ bincode = "1.3.3"
 rayon = "1.6.1"
 num_cpus = "1.14.0"
 num-format = "0.4.4"
+human_bytes = { version = "0.4", default-features = false }
 
 [profile.release]
 lto = true

diff --git a/src/data.rs b/src/data.rs
@@ -28,6 +28,9 @@ pub struct AppState {
     pub size: String,
     pub prob: String,
     #[data(ignore)]
+    pub indexed_data_in_bytes: u64,
+    pub indexed_data_in_bytes_string: String,
+    #[data(ignore)]
     pub settings: bool,
     pub properties: Vector<String>,
     pub view_column: String,
@@ -50,6 +53,7 @@ impl Drop for AppState {
 impl AppState {
     fn get_serializable_parameters(&self) -> SerializableParameters {
         SerializableParameters { view_column: self.view_column.to_string(),
+            indexed_data_in_bytes : self.indexed_data_in_bytes,
             pointer_state: self.pointers.iter().map(|p| p.clone()).collect::<Vec<PointerState>>()
         }
     }
@@ -59,6 +63,13 @@ impl AppState {
 pub struct SerializableParameters {
     pub view_column: String,
     pub pointer_state: Vec<PointerState>,
+    pub indexed_data_in_bytes: u64,
+}
+
+impl Default for SerializableParameters {
+    fn default() -> Self {
+        SerializableParameters { view_column: "".to_string(), indexed_data_in_bytes: 0, pointer_state: vec![] }
+    }
 }
 
 #[derive(Clone, Data, Lens, Serialize, Deserialize)]

diff --git a/src/index.rs b/src/index.rs
@@ -3,13 +3,14 @@ use std::collections::HashSet;
 use std::fs::File;
 use std::io::{BufRead, BufReader, Error, Read};
 use std::net::TcpListener;
-use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
 use std::thread::{JoinHandle, sleep};
 use std::time::{Duration, Instant};
 
 use bincode::deserialize;
 use crossbeam_channel::{Receiver, Sender};
 use druid::ExtEventSink;
+use human_bytes::human_bytes;
 use melt_rs::get_search_index;
 use melt_rs::index::SearchIndex;
 use num_format::{Locale, ToFormattedString};
@@ -20,6 +21,7 @@ use crate::data::AppState;
 
 pub static GLOBAL_COUNT: AtomicUsize = AtomicUsize::new(0);
 pub static GLOBAL_SIZE: AtomicUsize = AtomicUsize::new(0);
+pub static GLOBAL_DATA_SIZE: AtomicU64 = AtomicU64::new(0);
 
 pub fn search_thread(
     rx_search: Receiver<CommandMessage>,
@@ -147,7 +149,8 @@ fn index_tread(rx_search: Receiver<CommandMessage>, tx_res: Sender<ResultMessage
                         }
                         CommandMessage::InsertJson(cm) => {
                             let key = index.add(&cm);
-                            conn.put(key.to_le_bytes(), cm).unwrap();
+                            conn.put(key.to_le_bytes(), &cm).unwrap();
+                            GLOBAL_DATA_SIZE.fetch_add(cm.as_bytes().len() as u64, Ordering::SeqCst);
                             GLOBAL_COUNT.store(1 + key, Ordering::SeqCst);
                             if key % 100000 == 0 {
                                 GLOBAL_SIZE.store(index.get_size_bytes() / 1_000_000, Ordering::SeqCst);
@@ -157,6 +160,7 @@ fn index_tread(rx_search: Receiver<CommandMessage>, tx_res: Sender<ResultMessage
                             index.clear();
                             GLOBAL_SIZE.store(0, Ordering::SeqCst);
                             GLOBAL_COUNT.store(0, Ordering::SeqCst);
+                            GLOBAL_DATA_SIZE.store(0, Ordering::SeqCst);
                             let buf = dirs::home_dir().unwrap().into_os_string().into_string().unwrap();
                             let path = format!("{}/.melt.db", buf);
                             let _ = DB::destroy(&Options::default(), path);
@@ -186,6 +190,8 @@ fn socket_listener(tx_send: Sender<CommandMessage>, sink: ExtEventSink) {
             sink1.add_idle_callback(move |data: &mut AppState| {
                 data.count = format!("Documents    {}", GLOBAL_COUNT.load(Ordering::SeqCst).to_formatted_string(&Locale::en).to_string());
                 data.size = format!("Index size   {}", GLOBAL_SIZE.load(Ordering::SeqCst).to_formatted_string(&Locale::en).to_string());
+                data.indexed_data_in_bytes = GLOBAL_DATA_SIZE.load(Ordering::SeqCst);
+                data.indexed_data_in_bytes_string =format!("Data size    {}", human_bytes(GLOBAL_DATA_SIZE.load(Ordering::SeqCst) as f64));
             });
         }
     });

diff --git a/src/main.rs b/src/main.rs
@@ -13,6 +13,7 @@ unused_extern_crates
 use std::fs;
 use std::fs::File;
 use std::io::{Error, Read};
+use std::sync::atomic::Ordering::SeqCst;
 
 use bincode::deserialize;
 use crossbeam_channel::bounded;
@@ -25,7 +26,7 @@ use view::build_ui;
 
 use crate::data::SerializableParameters;
 use crate::delegate::Delegate;
-use crate::index::{CommandMessage, search_thread};
+use crate::index::{CommandMessage, GLOBAL_DATA_SIZE, search_thread};
 
 mod data;
 
@@ -60,6 +61,8 @@ pub fn main() {
         count: "0".to_string(),
         size: "0".to_string(),
         prob: "".to_string(),
+        indexed_data_in_bytes: parameters.indexed_data_in_bytes,
+        indexed_data_in_bytes_string: "".to_string(),
         settings: false,
         properties: Default::default(),
         view_column: parameters.view_column,
@@ -83,13 +86,16 @@ pub fn load_from_json() -> SerializableParameters {
     let file = get_file_as_byte_vec(&path);
     match file {
         Ok(file) => {
-            deserialize(&file).unwrap()
+            let parameters: SerializableParameters = deserialize(&file).unwrap_or(SerializableParameters::default());
+            GLOBAL_DATA_SIZE.store(parameters.indexed_data_in_bytes, SeqCst);
+            parameters
         }
         Err(_) => {
-            SerializableParameters{ view_column: "".to_string(), pointer_state: vec![] }
+            SerializableParameters::default()
         }
     }
 }
+
 fn get_file_as_byte_vec(filename: &String) -> Result<Vec<u8>, Error> {
     let mut f = File::open(&filename)?;
     let metadata = fs::metadata(&filename)?;

diff --git a/src/view.rs b/src/view.rs
@@ -88,6 +88,7 @@ pub fn build_ui() -> impl Widget<AppState> {
         .with_child(Label::raw().with_font(FontDescriptor::new(FontFamily::MONOSPACE)).lens(AppState::query_time).align_left())
         .with_child(Label::raw().with_font(FontDescriptor::new(FontFamily::MONOSPACE)).lens(AppState::count).align_left())
         .with_child(Label::raw().with_font(FontDescriptor::new(FontFamily::MONOSPACE)).lens(AppState::size).align_left())
+        .with_child(Label::raw().with_font(FontDescriptor::new(FontFamily::MONOSPACE)).lens(AppState::indexed_data_in_bytes_string).align_left())
         .with_child(Label::raw().with_font(FontDescriptor::new(FontFamily::MONOSPACE)).lens(AppState::prob).align_left())
         .with_child(Label::dynamic(|value: &AppState, _| {
             format!("Time limit   {:?} ms", value.timelimit as u64)