Merge pull request #56 from jjcomer/cache-errors

Improve head request system and add compression

Cargo.toml

@@ -5,7 +5,7 @@ doc = false
 
 [package]
 name = 'hogan'
-version = '0.9.0'
+version = '0.10.0'
 authors = [
     'Jonathan Morley <[email protected]>',
     'Josh Comer <[email protected]>',
@@ -37,8 +37,9 @@ futures = '0.3'
 parking_lot = '0.11'
 bincode = '1.3'
 lazy_static = '1'
-crossbeam-channel = '0.4'
-threadpool = '1.8'
+riker = '0.4'
+riker-patterns = '0.4'
+compression = '0.1'
 
 [dependencies.tokio]
 version = '0.2'
@@ -60,6 +61,10 @@ default-features = false
 version = '1'
 features = ['rc']
 
+[dependencies.uuid]
+version = '0.8'
+features = ['v4']
+
 [dev-dependencies]
 assert_cmd = '1.0'
 dir-diff = '0.3'

src/app/db.rs

@@ -1,4 +1,5 @@
 use anyhow::Result;
+use compression::prelude::*;
 use hogan::config::Environment;
 use rusqlite::{params, Connection, OpenFlags, NO_PARAMS};
 use serde::Deserialize;
@@ -35,7 +36,12 @@ pub fn read_sql_env(db_path: &str, env: &str, sha: &str) -> Result<Option<Enviro
     let data: Option<rusqlite::Result<Vec<u8>>> =
         query.query_map(params![key], |row| Ok(row.get(0)?))?.next();
     if let Some(data) = data {
-        let decoded: WritableEnvironment = match bincode::deserialize(&data?) {
+        let decompressed_data = data?
+            .iter()
+            .cloned()
+            .decode(&mut BZip2Decoder::new())
+            .collect::<Result<Vec<_>, _>>()?;
+        let decoded: WritableEnvironment = match bincode::deserialize(&decompressed_data) {
             Ok(environment) => environment,
             Err(e) => {
                 warn!("Unable to deserialize env: {} {:?}", key, e);
@@ -54,12 +60,22 @@ pub fn write_sql_env(db_path: &str, env: &str, sha: &str, data: &Environment) ->
     let key = gen_env_key(sha, env);
     let env_data: WritableEnvironment = data.into();
     let data = bincode::serialize(&env_data)?;
-
-    debug!("Writing to DB. Key: {} Size: {}", key, data.len());
+    let compressed_data = data
+        .iter()
+        .cloned()
+        .encode(&mut BZip2Encoder::new(6), Action::Finish)
+        .collect::<Result<Vec<_>, _>>()?;
+    debug!(
+        "Writing to DB. Key: {} Size: {} -> {} = {}",
+        key,
+        data.len(),
+        compressed_data.len(),
+        data.len() - compressed_data.len()
+    );
 
     conn.execute(
         "INSERT INTO hogan (key, data) VALUES (?1, ?2)",
-        params![key, data],
+        params![key, compressed_data],
     )
     .map_err(|e| e.into())
 }