From 72129b6e555d6542e89c8b3216ec1384e099bcf6 Mon Sep 17 00:00:00 2001 From: Josh Comer Date: Thu, 23 Nov 2023 15:30:18 -0400 Subject: [PATCH] Add redis caching and ability to configure db caches --- Cargo.lock | 402 ++++++++++++++++++++++-------------------- Cargo.toml | 8 +- src/app/config.rs | 8 +- src/app/server.rs | 48 +++-- src/git.rs | 4 +- src/main.rs | 2 + src/storage/cache.rs | 96 ++++++++++ src/storage/mod.rs | 1 + src/storage/redis.rs | 96 ++++++++++ src/storage/sqlite.rs | 107 ++--------- 10 files changed, 467 insertions(+), 305 deletions(-) create mode 100644 src/storage/redis.rs diff --git a/Cargo.lock b/Cargo.lock index 16254b9..b447c2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,7 +29,7 @@ dependencies = [ "actix-rt", "actix-service", "actix-utils", - "ahash 0.8.4", + "ahash 0.8.6", "base64", "bitflags 2.4.1", "brotli", @@ -65,7 +65,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote 1.0.33", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -77,7 +77,7 @@ dependencies = [ "bytestring", "http", "regex", - "serde 1.0.189", + "serde 1.0.193", "tracing", ] @@ -103,7 +103,7 @@ dependencies = [ "futures-core", "futures-util", "mio", - "socket2", + "socket2 0.5.5", "tokio", "tracing", ] @@ -144,7 +144,7 @@ dependencies = [ "actix-service", "actix-utils", "actix-web-codegen", - "ahash 0.8.4", + "ahash 0.8.6", "bytes", "bytestring", "cfg-if 1.0.0", @@ -160,11 +160,11 @@ dependencies = [ "once_cell", "pin-project-lite", "regex", - "serde 1.0.189", + "serde 1.0.193", "serde_json", "serde_urlencoded", "smallvec", - "socket2", + "socket2 0.5.5", "time", "url", ] @@ -178,7 +178,7 @@ dependencies = [ "actix-router", "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -218,12 +218,12 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72832d73be48bac96a5d7944568f305d829ed55b0ce3b483647089dfaf6cf704" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ "cfg-if 1.0.0", - "getrandom 0.2.10", + "getrandom 0.2.11", "once_cell", "version_check", "zerocopy", @@ -372,7 +372,7 @@ version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" dependencies = [ - "serde 1.0.189", + "serde 1.0.193", ] [[package]] @@ -409,9 +409,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -419,13 +419,13 @@ dependencies = [ [[package]] name = "bstr" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c79ad7fb2dd38f3dabd76b09c6a5a20c038fc0213ef1e9afd30eb777f120f019" +checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" dependencies = [ "memchr", "regex-automata", - "serde 1.0.189", + "serde 1.0.193", ] [[package]] @@ -448,9 +448,9 @@ checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "bytestring" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae" +checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" dependencies = [ "bytes", ] @@ -537,6 +537,16 @@ dependencies = [ "vec_map", ] +[[package]] +name = "combine" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "compression" version = "0.1.5" @@ -558,7 +568,7 @@ dependencies = [ "lazy_static", "nom", "rust-ini", - "serde 1.0.189", + "serde 1.0.193", "serde-hjson", "serde_json", "toml", @@ -567,23 +577,21 @@ dependencies = [ [[package]] name = "const-random" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e" +checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" dependencies = [ "const-random-macro", - "proc-macro-hack", ] [[package]] name = "const-random-macro" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.11", "once_cell", - "proc-macro-hack", "tiny-keccak", ] @@ -618,9 +626,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fbc60abd742b35f2492f808e1abbb83d45f72db402e14c55057edc9c7b1e9e4" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ "libc", ] @@ -769,11 +777,17 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" dependencies = [ "libc", "windows-sys", @@ -781,9 +795,9 @@ dependencies = [ [[package]] name = "fallible-iterator" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] name = "fallible-streaming-iterator" @@ -824,9 +838,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -839,9 +853,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" dependencies = [ "futures-channel", "futures-core", @@ -854,9 +868,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", "futures-sink", @@ -864,15 +878,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" dependencies = [ "futures-core", "futures-task", @@ -882,38 +896,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-channel", "futures-core", @@ -950,9 +964,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if 1.0.0", "libc", @@ -982,9 +996,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" dependencies = [ "bytes", "fnv", @@ -1009,23 +1023,17 @@ dependencies = [ "pest", "pest_derive", "quick-error", - "serde 1.0.189", + "serde 1.0.193", "serde_json", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" dependencies = [ - "ahash 0.8.4", + "ahash 0.8.6", "allocator-api2", ] @@ -1035,7 +1043,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.2", + "hashbrown", ] [[package]] @@ -1073,7 +1081,7 @@ dependencies = [ [[package]] name = "hogan" -version = "0.13.1" +version = "0.13.2" dependencies = [ "actix-web", "anyhow", @@ -1094,11 +1102,12 @@ dependencies = [ "lru_time_cache", "parking_lot", "predicates", + "redis", "regex", "riker", "riker-patterns", "rusqlite", - "serde 1.0.189", + "serde 1.0.193", "serde_derive", "serde_json", "shellexpand", @@ -1107,7 +1116,7 @@ dependencies = [ "tempfile", "thiserror", "url", - "uuid 1.5.0", + "uuid 1.6.1", "walkdir", "which", "zip", @@ -1124,9 +1133,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -1170,9 +1179,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1180,12 +1189,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.3" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ - "autocfg", - "hashbrown 0.12.3", + "equivalent", + "hashbrown", ] [[package]] @@ -1223,9 +1232,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" dependencies = [ "wasm-bindgen", ] @@ -1236,7 +1245,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55ff1e1486799e3f64129f8ccad108b38290df9cd7015cd31bed17239f0789d6" dependencies = [ - "serde 1.0.189", + "serde 1.0.193", "serde_json", "thiserror", "treediff", @@ -1269,9 +1278,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.149" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libgit2-sys" @@ -1287,11 +1296,22 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.1", + "libc", + "redox_syscall", +] + [[package]] name = "libsqlite3-sys" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" dependencies = [ "cc", "pkg-config", @@ -1342,15 +1362,15 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "local-channel" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a493488de5f18c8ffcba89eebb8532ffc562dc400490eb65b84893fae0b178" +checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8" dependencies = [ "futures-core", "futures-sink", @@ -1359,9 +1379,9 @@ dependencies = [ [[package]] name = "local-waker" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1" +checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" [[package]] name = "lock_api" @@ -1381,11 +1401,11 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "lru" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60" +checksum = "2994eeba8ed550fd9b47a0b38f0242bc3344e496483c6180b69139cc2fa5d1d7" dependencies = [ - "hashbrown 0.14.2", + "hashbrown", ] [[package]] @@ -1495,18 +1515,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.1.5+3.1.3" +version = "300.1.6+3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "559068e4c12950d7dcaa1857a61725c0d38d4fc03ff8e070ab31a75d6e316491" +checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" dependencies = [ "cc", "libc", @@ -1539,7 +1559,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.4.1", + "redox_syscall", "smallvec", "windows-targets", ] @@ -1575,15 +1595,15 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c022f1e7b65d6a24c0dbbd5fb344c66881bc01f3e5ae74a1c8100f2f985d98a4" +checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" dependencies = [ "memchr", "thiserror", @@ -1592,9 +1612,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35513f630d46400a977c4cb58f78e1bfbe01434316e60c37d27b9ad6139c66d8" +checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" dependencies = [ "pest", "pest_generator", @@ -1602,22 +1622,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc9fc1b9e7057baba189b5c626e2d6f40681ae5b6eb064dc7c7834101ec8123a" +checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" dependencies = [ "pest", "pest_meta", "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] name = "pest_meta" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df74e9e7ec4053ceb980e7c0c8bd3594e977fde1af91daba9c928e8e8c6708d" +checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" dependencies = [ "once_cell", "pest", @@ -1709,12 +1729,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "0.4.30" @@ -1816,7 +1830,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.11", ] [[package]] @@ -1829,21 +1843,18 @@ dependencies = [ ] [[package]] -name = "redox_syscall" -version = "0.2.16" +name = "redis" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba" dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags 1.3.2", + "combine", + "itoa", + "percent-encoding", + "ryu", + "sha1_smol", + "socket2 0.4.10", + "url", ] [[package]] @@ -1857,12 +1868,12 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" dependencies = [ - "getrandom 0.2.10", - "redox_syscall 0.2.16", + "getrandom 0.2.11", + "libredox", "thiserror", ] @@ -1939,9 +1950,9 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" +checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d" dependencies = [ "bitflags 2.4.1", "fallible-iterator", @@ -1974,9 +1985,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.20" +version = "0.38.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ce50cb2e16c2903e30d1cbccfd8387a74b9d4c938b6a4c5ec6cc7556f7a8a0" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" dependencies = [ "bitflags 2.4.1", "errno", @@ -2020,9 +2031,9 @@ checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" [[package]] name = "serde" -version = "1.0.189" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] @@ -2042,24 +2053,24 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.189" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", - "serde 1.0.189", + "serde 1.0.193", ] [[package]] @@ -2080,7 +2091,7 @@ dependencies = [ "form_urlencoded", "itoa", "ryu", - "serde 1.0.189", + "serde 1.0.193", ] [[package]] @@ -2094,6 +2105,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.10.8" @@ -2162,9 +2179,19 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.1" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" + +[[package]] +name = "socket2" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] [[package]] name = "socket2" @@ -2255,9 +2282,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.38" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2 1.0.69", "quote 1.0.33", @@ -2266,13 +2293,13 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if 1.0.0", "fastrand", - "redox_syscall 0.3.5", + "redox_syscall", "rustix", "windows-sys", ] @@ -2318,7 +2345,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -2340,7 +2367,7 @@ dependencies = [ "deranged", "itoa", "powerfmt", - "serde 1.0.189", + "serde 1.0.193", "time-core", "time-macros", ] @@ -2386,9 +2413,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ "backtrace", "bytes", @@ -2397,15 +2424,15 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.5", "windows-sys", ] [[package]] name = "tokio-util" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -2421,7 +2448,7 @@ version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ - "serde 1.0.189", + "serde 1.0.193", ] [[package]] @@ -2506,9 +2533,9 @@ checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -2521,16 +2548,16 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.11", ] [[package]] name = "uuid" -version = "1.5.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.11", ] [[package]] @@ -2584,9 +2611,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -2594,24 +2621,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.38", + "syn 2.0.39", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" dependencies = [ "quote 1.0.33", "wasm-bindgen-macro-support", @@ -2619,33 +2646,34 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.38", + "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" [[package]] name = "which" -version = "4.4.2" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +checksum = "9bf3ea8596f3a0dd5980b46430f2058dfe2c36a27ccfbb1845d6fbfcd9ba6e14" dependencies = [ "either", "home", "once_cell", "rustix", + "windows-sys", ] [[package]] @@ -2765,22 +2793,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.11" +version = "0.7.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c19fae0c8a9efc6a8281f2e623db8af1db9e57852e04cde3e754dd2dc29340f" +checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.11" +version = "0.7.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc56589e9ddd1f1c28d4b4b5c773ce232910a6bb67a70133d61c9e347585efe9" +checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" dependencies = [ "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 546bd17..5946055 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ doc = false [package] name = 'hogan' -version = '0.13.1' +version = '0.13.2' authors = [ 'Jonathan Morley ', 'Josh Comer ', @@ -27,6 +27,7 @@ log = '0.4' lru_time_cache = '0.11' lru = '0.12' parking_lot = '0.12' +redis = '0.23' riker = '0.4' riker-patterns = '0.4' serde_derive = '1.0' @@ -38,11 +39,12 @@ tempfile = '3' thiserror = '1.0' url = '2' walkdir = '2' -which = '4.4' +which = '5.0' zip = '0.6' + [dependencies.rusqlite] -version = '0.29' +version = '0.30' features = ['bundled'] [dependencies.git2] diff --git a/src/app/config.rs b/src/app/config.rs index ba5fb27..a2612d0 100644 --- a/src/app/config.rs +++ b/src/app/config.rs @@ -106,13 +106,17 @@ pub enum AppCommand { ///Filepath to the embedded db for storing environments. Will be created if it doesn't exist. If not provided a /// random temp directory will be created - #[structopt(long = "db", value_name = "PATH", default_value = "hogan.db")] - db_path: String, + #[structopt(long = "db", value_name = "PATH")] + db_path: Option, ///Maximum age of db entries based on number of days #[structopt(long = "db-max-age", value_name = "DAYS", default_value = "90")] db_max_age: usize, + ///Connection string for redis server. Off by default. Will use "db-max-age" as TTL value + #[structopt(long = "redis", value_name = "CONNECTION")] + redis_connection: Option, + ///The delay between background fetches against the git repo #[structopt( long = "fetch-poller", diff --git a/src/app/server.rs b/src/app/server.rs index dabe1b0..b11ebf3 100644 --- a/src/app/server.rs +++ b/src/app/server.rs @@ -3,13 +3,14 @@ use crate::app::datadogstatsd::{CustomMetrics, DdMetrics}; use crate::app::fetch_actor; use crate::app::head_actor; use crate::storage::cache::{Cache, CleanupActor}; -use crate::storage::{lru, multi, sqlite}; +use crate::storage::{lru, multi, redis, sqlite}; use actix_web::dev::Service; use actix_web::middleware::Logger; use actix_web::{get, middleware, post, web, HttpResponse, HttpServer}; use anyhow::{Context, Result}; use hogan::config::{ConfigDir, EnvironmentDescription}; use hogan::error::HoganError; +use itertools::Itertools; use parking_lot::Mutex; use regex::Regex; use riker::actors::ActorSystem; @@ -91,7 +92,8 @@ pub fn start_up_server( environments_regex: Regex, datadog: bool, environment_pattern: String, - db_path: String, + db_path: Option, + redis_connection: Option, db_max_age: usize, fetch_poller: u64, allow_fetch: bool, @@ -117,25 +119,43 @@ pub fn start_up_server( fetch_poller, ); - let cache = Arc::new(multi::MultiCache::new(vec![ - Box::new(lru::LruEnvCache::new("lru", cache_size)?), - Box::new(sqlite::SqliteCache::new(&db_path)), - ])); + info!("Adding LRU cache of size {}", cache_size); + let mut caches: Vec> = + vec![Box::new(lru::LruEnvCache::new("lru", cache_size)?)]; + + if let Some(connection_string) = redis_connection { + info!("Adding redis cache at: {}", connection_string); + caches.push(Box::new(redis::RedisCache::new( + "redis", + &connection_string, + db_max_age * 24 * 60 * 60, + )?)); + } - CleanupActor::init_db_cleanup_system( - &actor_system, - &[Arc::new(Box::new(sqlite::SqliteCache::new(&db_path)))], - db_max_age, - dd_metrics.clone(), - ); + if let Some(db_path) = db_path { + info!("Adding sqlite cache at: {}", db_path); + let sqlite_cache = Box::new(sqlite::SqliteCache::new(&db_path)); + caches.push(sqlite_cache.clone()); + + info!("Starting db cleanup system"); + CleanupActor::init_db_cleanup_system( + &actor_system, + &[Arc::new(sqlite_cache.clone())], + db_max_age, + dd_metrics.clone(), + ); + } + + let cache_order = caches.iter().map(|c| c.id()).join(", "); + info!("Caching order: {}", cache_order); + + let cache = Arc::new(multi::MultiCache::new(caches)); let write_lock = Mutex::new(0); info!("Starting server on {}:{}", address, port); let state = ServerState { - //environments, - //environment_listings, cache, config_dir, write_lock, diff --git a/src/git.rs b/src/git.rs index b28815b..4ab9f99 100644 --- a/src/git.rs +++ b/src/git.rs @@ -124,7 +124,7 @@ fn detach_head(repo: &Repository, sha: &str) -> Result<()> { } pub fn ext_fetch(path: &Path, remote: &str) -> Result<()> { - info!("Fetching {}", remote); + debug!("Fetching {}", remote); let mut fetch_cmd = Command::new("git") .current_dir(path.to_str().unwrap()) .args(["fetch", "--prune", remote]) @@ -135,7 +135,7 @@ pub fn ext_fetch(path: &Path, remote: &str) -> Result<()> { } pub fn ext_maintenance(path: &Path) -> Result<()> { - info!("Performing maintenance"); + debug!("Performing maintenance"); let mut maintenance_cmd = Command::new("git") .current_dir(path.to_str().unwrap()) .args(["maintenance", "run", "--auto"]) diff --git a/src/main.rs b/src/main.rs index 966f543..c763ed7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,6 +50,7 @@ fn main() -> Result<()> { db_max_age, fetch_poller, allow_fetch, + redis_connection, } => { server::start_up_server( common, @@ -60,6 +61,7 @@ fn main() -> Result<()> { datadog, environment_pattern, db_path, + redis_connection, db_max_age, fetch_poller, allow_fetch, diff --git a/src/storage/cache.rs b/src/storage/cache.rs index 8ab3c13..1d265cb 100644 --- a/src/storage/cache.rs +++ b/src/storage/cache.rs @@ -1,9 +1,12 @@ use crate::app::datadogstatsd::CustomMetrics; use crate::app::datadogstatsd::DdMetrics; use anyhow::Result; +use compression::prelude::*; use hogan::config::Environment; use hogan::config::EnvironmentDescription; use riker::actors::*; +use serde::Deserialize; +use serde::Serialize; use std::sync::Arc; use std::time::{Duration, SystemTime}; @@ -16,6 +19,99 @@ pub trait Cache { fn write_env_listing(&self, sha: &str, data: &[EnvironmentDescription]) -> Result<()>; } +#[derive(Default, Serialize, Deserialize, Debug)] +pub struct WritableEnvironment { + pub config_data: String, + pub environment: String, + pub environment_type: Option, +} + +impl From<&Environment> for WritableEnvironment { + fn from(environment: &Environment) -> Self { + WritableEnvironment { + config_data: environment.config_data.to_string(), + environment: environment.environment.to_owned(), + environment_type: environment.environment_type.to_owned(), + } + } +} + +impl From for Environment { + fn from(environment: WritableEnvironment) -> Self { + Environment { + config_data: serde_json::from_str(&environment.config_data).unwrap(), + environment: environment.environment.to_owned(), + environment_type: environment.environment_type.to_owned(), + } + } +} + +#[derive(Default, Serialize, Deserialize, Debug)] +pub struct WritableEnvironmentListing { + pub environments: Vec, +} + +impl From<&[EnvironmentDescription]> for WritableEnvironmentListing { + fn from(environments: &[EnvironmentDescription]) -> Self { + Self { + environments: environments.to_owned(), + } + } +} + +fn compress_data(data: Vec) -> Result> { + let compressed_data = data + .into_iter() + .encode(&mut BZip2Encoder::new(6), Action::Finish) + .collect::, _>>()?; + + Ok(compressed_data) +} + +fn decompress_data(data: Vec) -> Result> { + let decompressed_data = data + .into_iter() + .decode(&mut BZip2Decoder::new()) + .collect::, _>>()?; + + Ok(decompressed_data) +} + +pub fn serialize_env(data: &Environment) -> Result> { + let writable_data: WritableEnvironment = data.into(); + let encoded_data = bincode::serialize(&writable_data)?; + let compressed_data = compress_data(encoded_data)?; + + Ok(compressed_data) +} + +pub fn deserialize_env(data: Vec) -> Result { + let decompressed_data = decompress_data(data)?; + let decoded: WritableEnvironment = match bincode::deserialize(&decompressed_data) { + Ok(environment) => environment, + Err(e) => { + return Err(e.into()); + } + }; + Ok(decoded.into()) +} + +pub fn serialize_env_listing(data: &[EnvironmentDescription]) -> Result> { + let writable_data: WritableEnvironmentListing = data.into(); + let encoded_data = bincode::serialize(&writable_data)?; + let compressed_data = compress_data(encoded_data)?; + Ok(compressed_data) +} + +pub fn deserialize_env_listing(data: Vec) -> Result> { + let decompressed_data = decompress_data(data)?; + let decoded: WritableEnvironmentListing = match bincode::deserialize(&decompressed_data) { + Ok(environment) => environment, + Err(e) => return Err(e.into()), + }; + Ok(decoded.environments) +} + #[derive(Debug, Clone)] pub struct ExecuteCleanup {} diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 3fa96ae..32d1700 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -2,3 +2,4 @@ pub mod cache; pub mod lru; pub mod multi; pub mod sqlite; +pub mod redis; \ No newline at end of file diff --git a/src/storage/redis.rs b/src/storage/redis.rs new file mode 100644 index 0000000..d479e2c --- /dev/null +++ b/src/storage/redis.rs @@ -0,0 +1,96 @@ +use super::cache; +use crate::storage::cache::Cache; +use anyhow::Result; +use hogan::config::Environment; +use hogan::config::EnvironmentDescription; +use redis; + +use redis::Commands as _; +use std::sync::Arc; + +pub struct RedisCache { + client: redis::Client, + id: String, + ttl: usize, +} + +impl RedisCache { + pub fn new(id: &str, connection_string: &str, ttl: usize) -> Result { + let client = redis::Client::open(connection_string)?; + Ok(RedisCache { + client, + id: id.to_owned(), + ttl, + }) + } +} + +impl Cache for RedisCache { + fn id(&self) -> &str { + &self.id + } + + fn clean(&self, _max_age: usize) -> Result<()> { + Ok(()) + } + + fn read_env(&self, env: &str, sha: &str) -> Result>> { + let key = gen_env_key(sha, env); + let mut connection = self.client.get_connection()?; + let serialized_data: Vec = match connection.get(&key) { + Ok(data) => { + connection.expire(&key, self.ttl)?; + data + } + Err(_e) => return Ok(None), + }; + + let data = cache::deserialize_env(serialized_data)?; + + Ok(Some(Arc::new(data))) + } + + fn write_env(&self, env: &str, sha: &str, data: &Environment) -> Result<()> { + let key = gen_env_key(sha, env); + let serialized_data = cache::serialize_env(data)?; + let mut connection = self.client.get_connection()?; + + let opts = redis::SetOptions::default().with_expiration(redis::SetExpiry::EX(self.ttl)); + + connection.set_options(&key, serialized_data, opts)?; + debug!("Wrote env {} to redis.", key); + Ok(()) + } + + fn read_env_listing(&self, sha: &str) -> Result>>> { + let key = gen_env_listing_key(sha); + let mut connection = self.client.get_connection()?; + let serialized_data: Vec = match connection.get(&key) { + Ok(data) => { + connection.expire(&key, self.ttl)?; + data + } + Err(_e) => return Ok(None), + }; + let data = cache::deserialize_env_listing(serialized_data)?; + Ok(Some(Arc::new(data))) + } + + fn write_env_listing(&self, sha: &str, data: &[EnvironmentDescription]) -> Result<()> { + let key = gen_env_listing_key(sha); + let serialized_data = cache::serialize_env_listing(data)?; + let mut connection = self.client.get_connection()?; + + let opts = redis::SetOptions::default().with_expiration(redis::SetExpiry::EX(self.ttl)); + connection.set_options(key, serialized_data, opts)?; + Ok(()) + } +} + +fn gen_env_key(sha: &str, env: &str) -> String { + format!("env::{}::{}", sha, env) +} + +fn gen_env_listing_key(sha: &str) -> String { + format!("listing::{}", sha) +} diff --git a/src/storage/sqlite.rs b/src/storage/sqlite.rs index f9a1b4c..5886d30 100644 --- a/src/storage/sqlite.rs +++ b/src/storage/sqlite.rs @@ -1,12 +1,11 @@ #![allow(clippy::from_over_into)] use crate::storage::cache::Cache; + +use super::cache; use anyhow::Result; -use compression::prelude::*; use hogan::config::Environment; use hogan::config::EnvironmentDescription; use rusqlite::{params, Connection, OpenFlags}; -use serde::Deserialize; -use serde::Serialize; use std::sync::Arc; #[derive(Debug, Clone)] @@ -92,18 +91,8 @@ fn read_sql_env(db_path: &str, env: &str, sha: &str) -> Result>> = query.query_map(params![key], |row| row.get(0))?.next(); if let Some(data) = data { - let decompressed_data = data? - .into_iter() - .decode(&mut BZip2Decoder::new()) - .collect::, _>>()?; - let decoded: WritableEnvironment = match bincode::deserialize(&decompressed_data) { - Ok(environment) => environment, - Err(e) => { - warn!("Unable to deserialize env: {} {:?}", key, e); - return Err(e.into()); - } - }; - Ok(Some(Arc::new(decoded.into()))) + let deserialized_data = cache::deserialize_env(data?)?; + Ok(Some(Arc::new(deserialized_data))) } else { debug!("Unable to find {} in sqlite db", key); Ok(None) @@ -113,24 +102,11 @@ fn read_sql_env(db_path: &str, env: &str, sha: &str) -> Result Result { let conn = open_sql_db(db_path, false)?; let key = gen_env_key(sha, env); - let env_data: WritableEnvironment = data.into(); - let data = bincode::serialize(&env_data)?; - let data_len = data.len(); - let compressed_data = data - .into_iter() - .encode(&mut BZip2Encoder::new(6), Action::Finish) - .collect::, _>>()?; - debug!( - "Writing to DB. Key: {} Size: {} -> {} = {}", - key, - data_len, - compressed_data.len(), - data_len - compressed_data.len() - ); + let serialized_data = cache::serialize_env(data)?; conn.execute( "INSERT INTO hogan (key, data) VALUES (?1, ?2)", - params![key, compressed_data], + params![key, serialized_data], ) .map_err(|e| e.into()) } @@ -143,46 +119,6 @@ fn gen_env_listing_key(sha: &str) -> String { format!("!listing::{}", sha) } -#[derive(Default, Serialize, Deserialize, Debug)] -struct WritableEnvironment { - config_data: String, - environment: String, - environment_type: Option, -} - -impl From<&Environment> for WritableEnvironment { - fn from(environment: &Environment) -> Self { - WritableEnvironment { - config_data: environment.config_data.to_string(), - environment: environment.environment.to_owned(), - environment_type: environment.environment_type.to_owned(), - } - } -} - -impl From for Environment { - fn from(environment: WritableEnvironment) -> Self { - Environment { - config_data: serde_json::from_str(&environment.config_data).unwrap(), - environment: environment.environment.to_owned(), - environment_type: environment.environment_type.to_owned(), - } - } -} - -#[derive(Default, Serialize, Deserialize, Debug)] -struct WritableEnvironmentListing { - environments: Vec, -} - -impl From<&[EnvironmentDescription]> for WritableEnvironmentListing { - fn from(environments: &[EnvironmentDescription]) -> Self { - Self { - environments: environments.to_owned(), - } - } -} - fn write_sql_env_listing( db_path: &str, sha: &str, @@ -190,24 +126,11 @@ fn write_sql_env_listing( ) -> Result { let conn = open_sql_db(db_path, false)?; let key = gen_env_listing_key(sha); - let env_data: WritableEnvironmentListing = data.into(); - let data = bincode::serialize(&env_data)?; - let data_len = data.len(); - let compressed_data = data - .into_iter() - .encode(&mut BZip2Encoder::new(6), Action::Finish) - .collect::, _>>()?; - debug!( - "Writing to DB. Key: {} Size: {} -> {} = {}", - key, - data_len, - compressed_data.len(), - data_len - compressed_data.len() - ); + let serialized_data = cache::serialize_env_listing(data)?; conn.execute( "INSERT INTO hogan (key, data) VALUES (?1, ?2)", - params![key, compressed_data], + params![key, serialized_data], ) .map_err(|e| e.into()) } @@ -222,18 +145,8 @@ fn read_sql_env_listing( let data: Option>> = query.query_map(params![key], |row| row.get(0))?.next(); if let Some(data) = data { - let decompressed_data = data? - .into_iter() - .decode(&mut BZip2Decoder::new()) - .collect::, _>>()?; - let decoded: WritableEnvironmentListing = match bincode::deserialize(&decompressed_data) { - Ok(environment) => environment, - Err(e) => { - warn!("Unable to deserialize env: {} {:?}", key, e); - return Err(e.into()); - } - }; - Ok(Some(Arc::new(decoded.environments))) + let deserialized_data = cache::deserialize_env_listing(data?)?; + Ok(Some(Arc::new(deserialized_data))) } else { debug!("Unable to find {} in sqlite db", key); Ok(None)