From 45bd59456aedbbfd4a6966777ee7bacbab79c513 Mon Sep 17 00:00:00 2001 From: Roman Kudryashov Date: Sun, 31 Oct 2021 19:26:02 +0300 Subject: [PATCH] Implement monitoring of the application using Prometheus, Grafana, Alertmanager, cAdvisor. Implement broadcasting of messages from Redis Pub/Sub --- Cargo.lock | 333 +++++----- mongodb-redis/.env.local | 2 +- mongodb-redis/Cargo.toml | 4 +- mongodb-redis/Dockerfile | 8 +- mongodb-redis/docker-compose.override.yml | 54 +- mongodb-redis/docker-compose.yml | 9 +- .../monitoring/alertmanager/alertmanager.yml | 12 + .../provisioning/dashboards/providers.yml | 11 + .../dashboards/webapp_metrics.json | 616 ++++++++++++++++++ .../provisioning/datasources/datasources.yml | 8 + .../monitoring/prometheus/prometheus.yml | 20 + mongodb-redis/monitoring/prometheus/rules.yml | 10 + mongodb-redis/src/broadcaster.rs | 77 +++ mongodb-redis/src/handlers.rs | 53 +- mongodb-redis/src/main.rs | 49 +- mongodb-redis/src/metrics.rs | 26 + mongodb-redis/src/redis.rs | 31 +- mongodb-redis/src/services.rs | 38 +- 18 files changed, 1143 insertions(+), 218 deletions(-) create mode 100644 mongodb-redis/monitoring/alertmanager/alertmanager.yml create mode 100644 mongodb-redis/monitoring/grafana/provisioning/dashboards/providers.yml create mode 100644 mongodb-redis/monitoring/grafana/provisioning/dashboards/webapp_metrics.json create mode 100644 mongodb-redis/monitoring/grafana/provisioning/datasources/datasources.yml create mode 100644 mongodb-redis/monitoring/prometheus/prometheus.yml create mode 100644 mongodb-redis/monitoring/prometheus/rules.yml create mode 100644 mongodb-redis/src/broadcaster.rs create mode 100644 mongodb-redis/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 72688c7..5836d4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,15 +4,16 @@ version = 3 [[package]] name = "actix-codec" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d5dbeb2d9e51344cb83ca7cc170f1217f9fe25bfc50160e6e200b5c31c1019a" +checksum = "13895df506faee81e423febbae3a33b27fca71831b96bb3d60adf16ebcfea952" dependencies = [ "bitflags", "bytes", "futures-core", "futures-sink", "log", + "memchr", "pin-project-lite", "tokio", "tokio-util", @@ -29,7 +30,7 @@ dependencies = [ "actix-service", "actix-tls", "actix-utils", - "ahash 0.7.4", + "ahash 0.7.6", "base64", "bitflags", "brotli2", @@ -86,9 +87,9 @@ dependencies = [ [[package]] name = "actix-rt" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc7d7cd957c9ed92288a7c3c96af81fa5291f65247a76a34dac7b6af74e52ba0" +checksum = "4a0c218d0a17c120f10ee0c69c9f0c45d87319e8f66b1f065e8412b612fc3e24" dependencies = [ "actix-macros", "futures-core", @@ -97,9 +98,9 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.0.0-beta.5" +version = "2.0.0-beta.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26369215fcc3b0176018b3b68756a8bcc275bb000e6212e454944913a1f9bf87" +checksum = "7367665785765b066ad16e1086d26a087f696bc7c42b6f93004ced6cfcf1eeca" dependencies = [ "actix-rt", "actix-service", @@ -108,15 +109,14 @@ dependencies = [ "log", "mio", "num_cpus", - "slab", "tokio", ] [[package]] name = "actix-service" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f5f9d66a8730d0fae62c26f3424f5751e5518086628a40b7ab6fca4a705034" +checksum = "8d3dc6a618b082974a08d7a4781d24d4691cba51500059bfebe6656a61ebfe1e" dependencies = [ "futures-core", "paste", @@ -165,7 +165,7 @@ dependencies = [ "actix-service", "actix-utils", "actix-web-codegen", - "ahash 0.7.4", + "ahash 0.7.6", "bytes", "cfg-if 1.0.0", "cookie", @@ -186,7 +186,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2 0.4.1", + "socket2 0.4.2", "time 0.3.4", "url", ] @@ -217,9 +217,9 @@ checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" [[package]] name = "ahash" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ "getrandom", "once_cell", @@ -237,15 +237,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.42" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "595d3cfa7a60d4555cb5067b99f07142a08ea778de5cf993f7b75c7d8fabc486" +checksum = "ee10e43ae4a853c0a3591d4e2ada1719e553be18199d9da9d4a83f5927c2f5c7" [[package]] name = "arc-swap" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e906254e445520903e7fc9da4f709886c84ae4bc4ddaf0e093188d66df4dc820" +checksum = "e6df5aef5c5830360ce5218cecb8f018af3438af5686ae945094affc86fdec63" [[package]] name = "async-stream" @@ -321,9 +321,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da1976d75adbe5fbc88130ecd119529cf1cc6a93ae1546d8696ee66f0d21af1" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "block-buffer" @@ -360,7 +360,7 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff58d466782b57e0001c8e97c6a70c01c2359d7e13e257a83654c0b783ecc139" dependencies = [ - "ahash 0.7.4", + "ahash 0.7.6", "base64", "chrono", "hex", @@ -375,9 +375,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.7.0" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" +checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c" [[package]] name = "byteorder" @@ -387,9 +387,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" [[package]] name = "bytestring" @@ -402,9 +402,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.69" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2" +checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd" dependencies = [ "jobserver", ] @@ -446,15 +446,16 @@ dependencies = [ [[package]] name = "combine" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2d47c1b11006b87e492b53b313bb699ce60e16613c4dddaa91f8f7c220ab2fa" +checksum = "b2b2f5d0ee456f3928812dfc8c6d9a1d592b98678f6d56db9b0cd2b7bc6c8db5" dependencies = [ "bytes", - "futures-util", + "futures-core", "memchr", "pin-project-lite", "tokio", + "tokio-util", ] [[package]] @@ -482,9 +483,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3" dependencies = [ "core-foundation-sys", "libc", @@ -492,15 +493,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" -version = "0.1.5" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef" +checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" dependencies = [ "libc", ] @@ -689,9 +690,9 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "encoding_rs" -version = "0.8.28" +version = "0.8.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +checksum = "a74ea89a0a1b98f6332de42c95baff457ada66d1cb4030f9ff151b2041a1c746" dependencies = [ "cfg-if 1.0.0", ] @@ -748,9 +749,9 @@ checksum = "398ea4fabe40b9b0d885340a2a991a44c8a645624075ad966d21f88688e2b69e" [[package]] name = "flate2" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" +checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" dependencies = [ "cfg-if 1.0.0", "crc32fast", @@ -918,9 +919,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.3" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726" +checksum = "7fd819562fcebdac5afc5c113c3ec36f902840b70fd4fc458799c8ce4607ae55" dependencies = [ "bytes", "fnv", @@ -988,9 +989,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" +checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b" dependencies = [ "bytes", "fnv", @@ -1052,7 +1053,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.1", + "socket2 0.4.2", "tokio", "tower-service", "tracing", @@ -1122,9 +1123,9 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.10" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if 1.0.0", ] @@ -1158,24 +1159,24 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "jobserver" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5ca711fd837261e14ec9e674f092cbb931d3fa1482b017ae59328ddc6f3212b" +checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.52" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce791b7ca6638aae45be056e068fc756d871eb3b3b10b8efa62d1c9cec616752" +checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84" dependencies = [ "wasm-bindgen", ] @@ -1194,9 +1195,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.99" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7f823d141fe0a24df1e23b4af4e3c7ba9e5966ec514ea068c93024aa7deb765" +checksum = "a60553f9a9e039a333b4e9b20573b9e9b9c0bb3a11e201ccc48ef4283456d673" [[package]] name = "linked-hash-map" @@ -1233,9 +1234,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" +checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" dependencies = [ "scopeguard", ] @@ -1289,9 +1290,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" [[package]] name = "memoffset" @@ -1351,9 +1352,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" +checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" dependencies = [ "libc", "log", @@ -1401,7 +1402,7 @@ dependencies = [ "serde_with", "sha-1", "sha2", - "socket2 0.4.1", + "socket2 0.4.2", "stringprep", "strsim", "take_mut", @@ -1427,9 +1428,11 @@ dependencies = [ "derive_more", "dotenv", "env_logger 0.9.0", + "lazy_static", "log", "mime", "mongodb", + "prometheus", "redis", "rust-embed", "serde", @@ -1531,9 +1534,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.35" +version = "0.10.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885" +checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -1551,9 +1554,9 @@ checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" [[package]] name = "openssl-sys" -version = "0.9.65" +version = "0.9.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d" +checksum = "c6517987b3f8226b5da3661dad65ff7f300cc59fb5ea8333ca191fc65fde3edf" dependencies = [ "autocfg", "cc", @@ -1584,13 +1587,13 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", - "lock_api 0.4.4", - "parking_lot_core 0.8.3", + "lock_api 0.4.5", + "parking_lot_core 0.8.5", ] [[package]] @@ -1609,9 +1612,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" dependencies = [ "cfg-if 1.0.0", "instant", @@ -1695,15 +1698,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.19" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +checksum = "12295df4f294471248581bc09bef3c38a5e46f1e36d6a37353621a0c6c357e1f" [[package]] name = "ppv-lite86" -version = "0.2.10" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +checksum = "ed0cfbc8191465bed66e1718596ee0b0b35d5ee1f41c5df2189d0fe8bde535ba" [[package]] name = "pq-sys" @@ -1738,13 +1741,44 @@ checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" [[package]] name = "proc-macro2" -version = "1.0.28" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7ed8b8c7b886ea3ed7dde405212185f423ab44682667c8c6dd14aa1d9f6612" +checksum = "ba508cc11742c0dc5c1659771673afbab7a0efab23aa17e854cbab0837ed0b43" dependencies = [ "unicode-xid", ] +[[package]] +name = "procfs" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95e344cafeaeefe487300c361654bcfc85db3ac53619eeccced29f5ea18c4c70" +dependencies = [ + "bitflags", + "byteorder", + "flate2", + "hex", + "lazy_static", + "libc", +] + +[[package]] +name = "prometheus" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f64969ffd5dd8f39bd57a68ac53c163a095ed9d0fb707146da1b27025a3504" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "libc", + "memchr", + "parking_lot 0.11.2", + "procfs", + "protobuf", + "thiserror", +] + [[package]] name = "prost" version = "0.9.0" @@ -1798,6 +1832,12 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf" +version = "2.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47c327e191621a2158159df97cdbc2e7074bb4e940275e35abf38eb3d2595754" + [[package]] name = "quick-error" version = "1.2.3" @@ -1806,9 +1846,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" dependencies = [ "proc-macro2", ] @@ -1820,7 +1860,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" dependencies = [ "log", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "scheduled-thread-pool", ] @@ -1866,9 +1906,9 @@ dependencies = [ [[package]] name = "redis" -version = "0.21.3" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd71bdb3d0d6e9183e675c977f652fbf8abc3b63fcb722e9abb42f82ef839b65" +checksum = "7f23ceed4c0e76b322657c2c3352ea116f9ec60a1a1aefeb3c84ed062c50865b" dependencies = [ "arc-swap", "async-trait", @@ -2090,7 +2130,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" dependencies = [ - "parking_lot 0.11.1", + "parking_lot 0.11.2", ] [[package]] @@ -2111,9 +2151,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.3.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" +checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" dependencies = [ "bitflags", "core-foundation", @@ -2124,9 +2164,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.3.0" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284" +checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e" dependencies = [ "core-foundation-sys", "libc", @@ -2220,9 +2260,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "1.9.4" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad9fdbb69badc8916db738c25efd04f0a65297d26c2f8de4b62e57b8c12bc72" +checksum = "ad6056b4cb69b6e43e3a0f055def223380baecc99da683884f205bf347f7c4b3" dependencies = [ "rustversion", "serde", @@ -2231,9 +2271,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "1.4.2" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1569374bd54623ec8bd592cf22ba6e03c0f177ff55fbc8c29a49e296e7adecf" +checksum = "12e47be9471c72889ebafb5e14d5ff930d89ae7a67bbdb5f8abb564f845a927e" dependencies = [ "darling", "proc-macro2", @@ -2243,9 +2283,9 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a0c8611594e2ab4ebbf06ec7cbbf0a99450b8570e96cbf5188b5d5f6ef18d81" +checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" dependencies = [ "block-buffer", "cfg-if 1.0.0", @@ -2262,9 +2302,9 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "sha2" -version = "0.9.5" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" +checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" dependencies = [ "block-buffer", "cfg-if 1.0.0", @@ -2284,15 +2324,15 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" +checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" [[package]] name = "smallvec" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" +checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309" [[package]] name = "socket2" @@ -2307,9 +2347,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765f090f0e423d2b55843402a07915add955e7d60657db13707a159727326cad" +checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516" dependencies = [ "libc", "winapi", @@ -2454,9 +2494,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.74" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c" +checksum = "f2afee18b8beb5a596ecb4a2dce128c719b4ba399d34126b9e4396e3f9860966" dependencies = [ "proc-macro2", "quote", @@ -2496,9 +2536,9 @@ dependencies = [ [[package]] name = "teloxide-core" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "861d52d8f1a95698f3da02360cd2456a543f09ce6822e4f03d2a47c9ed3c116e" +checksum = "114c9057a3a2f74d937ece64029b362f583a69fb4b7405722c6dc03cd5bb4658" dependencies = [ "bytes", "chrono", @@ -2557,18 +2597,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.26" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93119e4feac1cbe6c798c34d3a53ea0026b0b1de6a120deef895137c0529bfe2" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.26" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "060d69a0afe7796bf42e9e2ff91f5ee691fb15c53d38b4b62a9a53eb23164745" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" dependencies = [ "proc-macro2", "quote", @@ -2636,9 +2676,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.3.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848a1e1181b9f6753b5e96a092749e29b11d19ede67dfbbd6c7dc7e0f49b5338" +checksum = "f83b2a3d4d9091d0abd7eba4dc2710b1718583bd4d8992e2190720ea38f391f7" dependencies = [ "tinyvec_macros", ] @@ -2662,7 +2702,7 @@ dependencies = [ "mio", "num_cpus", "once_cell", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "pin-project-lite", "signal-hook-registry", "tokio-macros", @@ -2681,9 +2721,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.3.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54473be61f4ebe4efd09cec9bd5d16fa51d70ea0192213d754d2d500457db110" +checksum = "114383b041aa6212c579467afa0075fbbdd0718de036100bc0ba7961d8cb9095" dependencies = [ "proc-macro2", "quote", @@ -2724,9 +2764,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.7" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" +checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" dependencies = [ "bytes", "futures-core", @@ -2781,14 +2821,15 @@ dependencies = [ [[package]] name = "tower" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60422bc7fefa2f3ec70359b8ff1caff59d785877eb70595904605bcc412470f" +checksum = "c00e500fff5fa1131c866b246041a6bf96da9c965f8fe4128cb1421f23e93c00" dependencies = [ "futures-core", "futures-util", "indexmap", "pin-project", + "pin-project-lite", "rand", "slab", "tokio", @@ -2813,9 +2854,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.26" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" +checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" dependencies = [ "cfg-if 1.0.0", "log", @@ -2826,9 +2867,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.15" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42e6fa53307c8a17e4ccd4dc81cf5ec38db9209f59b222210375b54ee40d1e2" +checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e" dependencies = [ "proc-macro2", "quote", @@ -2837,9 +2878,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.18" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" +checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" dependencies = [ "lazy_static", ] @@ -2891,7 +2932,7 @@ dependencies = [ "lazy_static", "log", "lru-cache", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "resolv-conf", "smallvec", "thiserror", @@ -2907,9 +2948,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "typed-builder" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345426c7406aa355b60c5007c79a2d1f5b605540072795222f17f6443e6a9c6f" +checksum = "a46ee5bd706ff79131be9c94e7edcb82b703c487766a114434e5790361cf08c5" dependencies = [ "proc-macro2", "quote", @@ -2918,9 +2959,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" +checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" [[package]] name = "ucd-trie" @@ -2939,9 +2980,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" [[package]] name = "unicode-normalization" @@ -3039,9 +3080,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.75" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b608ecc8f4198fe8680e2ed18eccab5f0cd4caaf3d83516fa5fb2e927fda2586" +checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -3049,9 +3090,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.75" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "580aa3a91a63d23aac5b6b267e2d13cb4f363e31dce6c352fca4752ae12e479f" +checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" dependencies = [ "bumpalo", "lazy_static", @@ -3064,9 +3105,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.25" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16646b21c3add8e13fdb8f20172f8a28c3dbf62f45406bcff0233188226cfe0c" +checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3076,9 +3117,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.75" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171ebf0ed9e1458810dfcb31f2e766ad6b3a89dbda42d8901f2b268277e5f09c" +checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3086,9 +3127,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.75" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c2657dd393f03aa2a659c25c6ae18a13a4048cebd220e147933ea837efc589f" +checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" dependencies = [ "proc-macro2", "quote", @@ -3099,15 +3140,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.75" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e0c4a743a309662d45f4ede961d7afa4ba4131a59a639f29b0069c3798bbcc2" +checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" [[package]] name = "web-sys" -version = "0.3.52" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01c70a82d842c9979078c772d4a1344685045f1a5628f677c2b2eab4dd7d2696" +checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/mongodb-redis/.env.local b/mongodb-redis/.env.local index 1e81105..cefbd8c 100644 --- a/mongodb-redis/.env.local +++ b/mongodb-redis/.env.local @@ -3,4 +3,4 @@ MONGODB_URI=mongodb://admin:password@localhost:27017 REDIS_URI=redis://localhost RUST_LOG=debug # defines whether to enable REST C(R)UD methods -ENABLE_WRITE_HANDLERS=true +ENABLE_WRITING_HANDLERS=true diff --git a/mongodb-redis/Cargo.toml b/mongodb-redis/Cargo.toml index 254d965..e15bbeb 100644 --- a/mongodb-redis/Cargo.toml +++ b/mongodb-redis/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [dependencies] mongodb = "2.0.1" -redis = { version = "0.21.3", features = ["tokio-comp", "connection-manager"] } +redis = { version = "0.21.4", features = ["tokio-comp", "connection-manager"] } actix-web = "4.0.0-beta.10" tokio = "1.13.0" tokio-stream = "0.1.8" @@ -19,3 +19,5 @@ log = "0.4.14" env_logger = "0.9.0" rust-embed = "6.2.0" mime = "0.3.16" +prometheus = { version = "0.13.0", features = ["process"] } +lazy_static = "1.4.0" diff --git a/mongodb-redis/Dockerfile b/mongodb-redis/Dockerfile index 83a1fc2..e8d05fd 100644 --- a/mongodb-redis/Dockerfile +++ b/mongodb-redis/Dockerfile @@ -2,9 +2,15 @@ FROM rust:1.56 ENV CARGO_TERM_COLOR always -WORKDIR /usr/src/app/docker-build +# create empty project for caching dependencies +RUN USER=root cargo new --bin /mongodb-redis/docker-build +WORKDIR /mongodb-redis/docker-build COPY /Cargo.lock ./ +COPY /mongodb-redis/Cargo.toml ./ +# cache dependencies +RUN cargo install --path . --locked COPY /mongodb-redis/ ./ +RUN touch ./src/main.rs RUN cargo install --path . --locked FROM debian:buster-slim diff --git a/mongodb-redis/docker-compose.override.yml b/mongodb-redis/docker-compose.override.yml index 8b87453..dc29a17 100644 --- a/mongodb-redis/docker-compose.override.yml +++ b/mongodb-redis/docker-compose.override.yml @@ -1,4 +1,4 @@ -# development configuration +# development configuration + services for monitoring version: '3.8' services: @@ -7,4 +7,54 @@ services: context: .. dockerfile: ./mongodb-redis/Dockerfile environment: - ENABLE_WRITE_HANDLERS: 'true' + ENABLE_WRITING_HANDLERS: 'true' + + # services for monitoring + # comment services below if you don't need monitoring + prometheus: + image: prom/prometheus:latest + container_name: prometheus + restart: always + ports: + - '9090:9090' + volumes: + - ./monitoring/prometheus:/etc/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--web.external-url=http://localhost:9090' + + grafana: + image: grafana/grafana:latest + container_name: grafana + restart: always + ports: + - '3000:3000' + volumes: + - ./monitoring/grafana/data:/var/lib/grafana + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + + alertmanager: + image: prom/alertmanager:latest + container_name: alertmanager + ports: + - '9093:9093' + volumes: + - ./monitoring/alertmanager:/etc/alertmanager + command: + - '--config.file=/etc/alertmanager/alertmanager.yml' + - '--web.external-url=http://localhost:9093' + + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + container_name: cadvisor + restart: always + ports: + - '8080:8080' + volumes: + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro diff --git a/mongodb-redis/docker-compose.yml b/mongodb-redis/docker-compose.yml index ed8062d..686ac2a 100644 --- a/mongodb-redis/docker-compose.yml +++ b/mongodb-redis/docker-compose.yml @@ -13,9 +13,9 @@ services: MONGODB_URI: mongodb://$MONGODB_USERNAME:$MONGODB_PASSWORD@mongodb:27017 REDIS_URI: redis://redis RUST_LOG: debug - ENABLE_WRITE_HANDLERS: 'false' + ENABLE_WRITING_HANDLERS: 'false' ports: - - 9000:9000 + - '9000:9000' mongodb: image: mongo:5 @@ -26,7 +26,7 @@ services: MONGO_INITDB_ROOT_PASSWORD: $MONGODB_PASSWORD MONGO_INITDB_DATABASE: solar_system_info ports: - - 27017:27017 + - '27017:27017' mongodb-seed: image: mongo:5 @@ -43,5 +43,6 @@ services: redis: image: redis:alpine container_name: redis + restart: always ports: - - 6379:6379 + - '6379:6379' diff --git a/mongodb-redis/monitoring/alertmanager/alertmanager.yml b/mongodb-redis/monitoring/alertmanager/alertmanager.yml new file mode 100644 index 0000000..8f12e65 --- /dev/null +++ b/mongodb-redis/monitoring/alertmanager/alertmanager.yml @@ -0,0 +1,12 @@ +route: + receiver: gmail + +receivers: +- name: gmail + email_configs: + - to: recipient@gmail.com + from: email_id@gmail.com + smarthost: smtp.gmail.com:587 + auth_username: email_id@gmail.com + auth_identity: email_id@gmail.com + auth_password: password diff --git a/mongodb-redis/monitoring/grafana/provisioning/dashboards/providers.yml b/mongodb-redis/monitoring/grafana/provisioning/dashboards/providers.yml new file mode 100644 index 0000000..e67b694 --- /dev/null +++ b/mongodb-redis/monitoring/grafana/provisioning/dashboards/providers.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: 'default' + folder: 'default' + type: file + allowUiUpdates: true + updateIntervalSeconds: 30 + options: + path: /etc/grafana/provisioning/dashboards + foldersFromFilesStructure: true diff --git a/mongodb-redis/monitoring/grafana/provisioning/dashboards/webapp_metrics.json b/mongodb-redis/monitoring/grafana/provisioning/dashboards/webapp_metrics.json new file mode 100644 index 0000000..3fee1ca --- /dev/null +++ b/mongodb-redis/monitoring/grafana/provisioning/dashboards/webapp_metrics.json @@ -0,0 +1,616 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 14, + "panels": [], + "title": "Web application metrics", + "type": "row" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 1, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(http_requests_total[5m])", + "interval": "", + "legendFormat": "{{method}} {{path}}", + "refId": "A" + } + ], + "title": "HTTP requests per second", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.4", + "targets": [ + { + "exemplar": true, + "expr": "rate(http_response_time_seconds_sum[5m]) / rate(http_response_time_seconds_count[5m])", + "interval": "", + "legendFormat": "{{method}} {{path}}", + "refId": "A" + } + ], + "title": "Mean response time", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 12 + }, + "id": 18, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "8.1.4", + "targets": [ + { + "exemplar": true, + "expr": "increase(http_requests_total{method=\"GET\",path=\"/planets\"}[1m])", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Requests number for the last minute", + "type": "gauge" + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateBlues", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": null, + "gridPos": { + "h": 12, + "w": 21, + "x": 3, + "y": 12 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 16, + "interval": null, + "legend": { + "show": false + }, + "maxDataPoints": null, + "pluginVersion": "8.1.4", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(increase(http_response_time_seconds_bucket{path=\"/planets\"}[30s])) by (le)", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "Response time distribution over time (`GET /planets`)", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": 10, + "xBucketSize": null, + "yAxis": { + "decimals": 2, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": 10, + "yBucketSize": null + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 18 + }, + "id": 2, + "interval": "5s", + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "8.1.4", + "targets": [ + { + "exemplar": true, + "expr": "http_connected_sse_clients", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Connected SSE clients", + "type": "gauge" + }, + { + "collapsed": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 12, + "panels": [], + "title": "System metrics", + "type": "row" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 4, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(process_cpu_seconds_total{job=\"mongodb_redis_web_app\"}[1m])", + "interval": "", + "legendFormat": "process", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{name='mongodb-redis'}[1m])) by (name)", + "interval": "", + "legendFormat": "container", + "refId": "B" + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisLabel": "MiB", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 5, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "process_resident_memory_bytes{job=\"mongodb_redis_web_app\"} / 1024 / 1024", + "interval": "", + "legendFormat": "process", + "refId": "A" + }, + { + "exemplar": true, + "expr": "container_memory_usage_bytes{name=\"mongodb-redis\"} / 1024 / 1024", + "interval": "", + "legendFormat": "container", + "refId": "B" + } + ], + "title": "Memory usage", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 1, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m" + ] + }, + "timezone": "", + "title": "webapp_metrics", + "uid": "7wcZ94dnz", + "version": 1 +} diff --git a/mongodb-redis/monitoring/grafana/provisioning/datasources/datasources.yml b/mongodb-redis/monitoring/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..1cdd49e --- /dev/null +++ b/mongodb-redis/monitoring/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: prometheus:9090 + isDefault: true diff --git a/mongodb-redis/monitoring/prometheus/prometheus.yml b/mongodb-redis/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..679cfc1 --- /dev/null +++ b/mongodb-redis/monitoring/prometheus/prometheus.yml @@ -0,0 +1,20 @@ +scrape_configs: + + - job_name: mongodb_redis_web_app + scrape_interval: 5s + static_configs: + # if this is not working when you're running the application in a container (not on a host machine) try to use 'mongodb-redis:9000' + - targets: ['host.docker.internal:9000'] + + - job_name: cadvisor + scrape_interval: 5s + static_configs: + - targets: ['cadvisor:8080'] + +rule_files: + - 'rules.yml' + +alerting: + alertmanagers: + - static_configs: + - targets: ['alertmanager:9093'] diff --git a/mongodb-redis/monitoring/prometheus/rules.yml b/mongodb-redis/monitoring/prometheus/rules.yml new file mode 100644 index 0000000..4fdda6b --- /dev/null +++ b/mongodb-redis/monitoring/prometheus/rules.yml @@ -0,0 +1,10 @@ +groups: +- name: default + rules: + - alert: SseClients + expr: http_connected_sse_clients > 0 + for: 1m + labels: + severity: high + annotations: + summary: Too many SSE clients diff --git a/mongodb-redis/src/broadcaster.rs b/mongodb-redis/src/broadcaster.rs new file mode 100644 index 0000000..94111a3 --- /dev/null +++ b/mongodb-redis/src/broadcaster.rs @@ -0,0 +1,77 @@ +use std::sync::Mutex; +use std::time::Duration; + +use actix_web::web::{Bytes, Data}; +use tokio::sync::mpsc::{self, Receiver, Sender}; +use tokio::{task, time}; + +use crate::errors::CustomError; + +#[derive(Clone)] +pub struct Broadcaster { + clients: Vec>>, +} + +impl Broadcaster { + fn new() -> Self { + Broadcaster { + clients: Vec::new(), + } + } + + pub fn create() -> Data> { + let me = Data::new(Mutex::new(Broadcaster::new())); + + // ping clients every 10 seconds to see if they are alive + Broadcaster::spawn_ping(me.clone()); + + me + } + + pub async fn new_client(&mut self) -> Receiver> { + let (tx, rx) = mpsc::channel::>(100); + + tx.clone() + .send(Ok(Bytes::from("data: Connected\n\n"))) + .await + .expect("Can't create a client"); + + self.clients.push(tx); + crate::metrics::HTTP_CONNECTED_SSE_CLIENTS.inc(); + rx + } + + pub fn send(&self, msg: Bytes) { + for client in self.clients.iter() { + client + .try_send(Ok(msg.clone())) + .expect("Can't send a message"); + } + } + + fn spawn_ping(me: Data>) { + task::spawn(async move { + let mut interval = time::interval(Duration::from_secs(10)); + + loop { + interval.tick().await; + let mut broadcaster_mutex = me.lock().expect("Failed to lock broadcaster"); + broadcaster_mutex.remove_stale_clients(); + crate::metrics::HTTP_CONNECTED_SSE_CLIENTS + .set(broadcaster_mutex.clients.len() as i64) + } + }); + } + + fn remove_stale_clients(&mut self) { + let mut ok_clients = Vec::new(); + for client in self.clients.iter() { + let result = client.clone().try_send(Ok(Bytes::from("data: Ping\n\n"))); + + if let Ok(()) = result { + ok_clients.push(client.clone()); + } + } + self.clients = ok_clients; + } +} diff --git a/mongodb-redis/src/handlers.rs b/mongodb-redis/src/handlers.rs index a38eabb..794e8b7 100644 --- a/mongodb-redis/src/handlers.rs +++ b/mongodb-redis/src/handlers.rs @@ -1,13 +1,15 @@ -use std::sync::Arc; +use actix_web::http::header::{self, ContentType}; +use actix_web::http::StatusCode; +use actix_web::{web, HttpRequest, HttpResponse}; +use prometheus::{Encoder, TextEncoder}; +use serde::Deserialize; +use crate::broadcaster::Broadcaster; use crate::dto::PlanetDto; use crate::errors::CustomError; use crate::model::PlanetType; use crate::services::{PlanetService, RateLimitingService}; -use actix_web::http::header::{self, ContentType}; -use actix_web::http::StatusCode; -use actix_web::{web, HttpRequest, HttpResponse}; -use serde::Deserialize; +use std::sync::Mutex; #[derive(Debug, Deserialize)] pub struct GetPlanetsQueryParams { @@ -17,8 +19,8 @@ pub struct GetPlanetsQueryParams { pub async fn get_planets( req: HttpRequest, web::Query(query_params): web::Query, - rate_limit_service: web::Data>, - planet_service: web::Data>, + rate_limit_service: web::Data, + planet_service: web::Data, ) -> Result { // can be moved to actix middleware rate_limit_service @@ -31,7 +33,7 @@ pub async fn get_planets( pub async fn create_planet( planet_dto: web::Json, - planet_service: web::Data>, + planet_service: web::Data, ) -> Result { let planet = planet_service .create_planet(planet_dto.into_inner().into()) @@ -42,7 +44,7 @@ pub async fn create_planet( pub async fn get_planet( planet_id: web::Path, - planet_service: web::Data>, + planet_service: web::Data, ) -> Result { let planet = planet_service.get_planet(&planet_id.into_inner()).await?; Ok(HttpResponse::Ok().json(PlanetDto::from(planet))) @@ -51,7 +53,7 @@ pub async fn get_planet( pub async fn update_planet( planet_id: web::Path, planet_dto: web::Json, - planet_service: web::Data>, + planet_service: web::Data, ) -> Result { let planet = planet_service .update_planet(&planet_id.into_inner(), planet_dto.into_inner().into()) @@ -62,7 +64,7 @@ pub async fn update_planet( pub async fn delete_planet( planet_id: web::Path, - planet_service: web::Data>, + planet_service: web::Data, ) -> Result { planet_service .delete_planet(&planet_id.into_inner()) @@ -73,7 +75,7 @@ pub async fn delete_planet( pub async fn get_image_of_planet( planet_id: web::Path, - planet_service: web::Data>, + planet_service: web::Data, ) -> Result { let image = planet_service .get_image_of_planet(&planet_id.into_inner()) @@ -84,11 +86,13 @@ pub async fn get_image_of_planet( .body(image)) } -pub async fn sse( - planet_service: web::Data>, -) -> Result { - let new_planets_stream = planet_service.get_new_planets_stream().await?; - let response_stream = tokio_stream::wrappers::ReceiverStream::new(new_planets_stream); +pub async fn sse(broadcaster: web::Data>) -> Result { + let rx = broadcaster + .lock() + .expect("Can't lock broadcaster") + .new_client() + .await; + let response_stream = tokio_stream::wrappers::ReceiverStream::new(rx); Ok(HttpResponse::build(StatusCode::OK) .insert_header(header::ContentType(mime::TEXT_EVENT_STREAM)) @@ -103,6 +107,21 @@ pub async fn index() -> Result { .body(content)) } +pub async fn metrics() -> Result { + let encoder = TextEncoder::new(); + let mut buffer = vec![]; + encoder + .encode(&prometheus::gather(), &mut buffer) + .expect("Failed to encode metrics"); + + let response = String::from_utf8(buffer.clone()).expect("Failed to convert bytes to string"); + buffer.clear(); + + Ok(HttpResponse::Ok() + .insert_header(header::ContentType(mime::TEXT_PLAIN)) + .body(response)) +} + fn get_ip_addr(req: &HttpRequest) -> Result { Ok(req .peer_addr() diff --git a/mongodb-redis/src/main.rs b/mongodb-redis/src/main.rs index c99a57d..33f1a29 100644 --- a/mongodb-redis/src/main.rs +++ b/mongodb-redis/src/main.rs @@ -1,16 +1,20 @@ use std::env; -use std::sync::Arc; +use actix_web::dev::Service; +use actix_web::web::Data; use actix_web::{web, App, HttpServer}; use log::info; +use crate::broadcaster::Broadcaster; use crate::db::MongoDbClient; use crate::services::{PlanetService, RateLimitingService}; +mod broadcaster; mod db; mod dto; mod errors; mod handlers; +mod metrics; mod model; mod redis; mod services; @@ -34,20 +38,45 @@ async fn main() -> std::io::Result<()> { .await .expect("Can't create Redis connection manager"); - let planet_service = Arc::new(PlanetService::new( + let broadcaster = Broadcaster::create(); + + redis::start_pubsub(&redis_client, broadcaster.clone()) + .await + .expect("Can't start Redis Pub/Sub"); + + let planet_service = Data::new(PlanetService::new( mongodb_client, redis_client, redis_connection_manager.clone(), )); - let rate_limiting_service = Arc::new(RateLimitingService::new(redis_connection_manager)); - let enable_write_handlers = env::var("ENABLE_WRITE_HANDLERS") - .expect("ENABLE_WRITE_HANDLERS env var should be specified") + let rate_limiting_service = Data::new(RateLimitingService::new(redis_connection_manager)); + + let enable_writing_handlers = env::var("ENABLE_WRITING_HANDLERS") + .expect("ENABLE_WRITING_HANDLERS env var should be specified") .parse::() - .expect("Can't parse ENABLE_WRITE_HANDLERS"); + .expect("Can't parse ENABLE_WRITING_HANDLERS"); HttpServer::new(move || { let mut app = App::new() + .wrap_fn(|req, srv| { + let request_method = req.method().to_string(); + let request_path = req.path(); + let histogram_timer = metrics::HTTP_RESPONSE_TIME_SECONDS + .with_label_values(&[&request_method, request_path]) + .start_timer(); + metrics::HTTP_REQUESTS_TOTAL + .with_label_values(&[&request_method, request_path]) + .inc(); + + let fut = srv.call(req); + + async { + let res = fut.await?; + histogram_timer.observe_duration(); + Ok(res) + } + }) .route("/planets", web::get().to(handlers::get_planets)) .route("/planets/{planet_id}", web::get().to(handlers::get_planet)) .route( @@ -56,10 +85,12 @@ async fn main() -> std::io::Result<()> { ) .route("/events", web::get().to(handlers::sse)) .route("/", web::get().to(handlers::index)) - .data(Arc::clone(&planet_service)) - .data(Arc::clone(&rate_limiting_service)); + .route("/metrics", web::get().to(handlers::metrics)) + .app_data(planet_service.clone()) + .app_data(rate_limiting_service.clone()) + .app_data(broadcaster.clone()); - if enable_write_handlers { + if enable_writing_handlers { app = app .route("/planets", web::post().to(handlers::create_planet)) .route( diff --git a/mongodb-redis/src/metrics.rs b/mongodb-redis/src/metrics.rs new file mode 100644 index 0000000..4f02c61 --- /dev/null +++ b/mongodb-redis/src/metrics.rs @@ -0,0 +1,26 @@ +use lazy_static::lazy_static; +use prometheus::{opts, register_histogram_vec, register_int_counter_vec, register_int_gauge}; +use prometheus::{HistogramVec, IntCounterVec, IntGauge}; + +const HTTP_RESPONSE_TIME_CUSTOM_BUCKETS: &[f64; 14] = &[ + 0.0005, 0.0008, 0.00085, 0.0009, 0.00095, 0.001, 0.00105, 0.0011, 0.00115, 0.0012, 0.0015, + 0.002, 0.003, 1.0, +]; + +lazy_static! { + pub static ref HTTP_REQUESTS_TOTAL: IntCounterVec = register_int_counter_vec!( + opts!("http_requests_total", "HTTP requests total"), + &["method", "path"] + ) + .expect("Can't create a metric"); + pub static ref HTTP_CONNECTED_SSE_CLIENTS: IntGauge = + register_int_gauge!(opts!("http_connected_sse_clients", "Connected SSE clients")) + .expect("Can't create a metric"); + pub static ref HTTP_RESPONSE_TIME_SECONDS: HistogramVec = register_histogram_vec!( + "http_response_time_seconds", + "HTTP response times", + &["method", "path"], + HTTP_RESPONSE_TIME_CUSTOM_BUCKETS.to_vec() + ) + .expect("Can't create a metric"); +} diff --git a/mongodb-redis/src/redis.rs b/mongodb-redis/src/redis.rs index 15e3b7b..f568e0d 100644 --- a/mongodb-redis/src/redis.rs +++ b/mongodb-redis/src/redis.rs @@ -1,12 +1,41 @@ -use redis::{Client, RedisError}; +use actix_web::web::{Bytes, Data}; +use redis::{Client, FromRedisValue, RedisError}; use redis::{RedisWrite, ToRedisArgs}; +use tokio_stream::StreamExt; +use crate::broadcaster::Broadcaster; +use crate::errors::CustomError; use crate::model::Planet; +use crate::services::NEW_PLANETS_CHANNEL_NAME; +use std::sync::Mutex; pub async fn create_client(redis_uri: String) -> Result { Ok(Client::open(redis_uri)?) } +pub async fn start_pubsub( + redis_client: &Client, + broadcaster: Data>, +) -> Result<(), CustomError> { + let mut pubsub_con = redis_client.get_async_connection().await?.into_pubsub(); + pubsub_con.subscribe(NEW_PLANETS_CHANNEL_NAME).await?; + + tokio::spawn(async move { + while let Some(msg) = pubsub_con.on_message().next().await { + let payload = msg.get_payload().expect("Can't get payload of message"); + let payload: String = + FromRedisValue::from_redis_value(&payload).expect("Can't convert from Redis value"); + let msg = Bytes::from(format!("data: Planet created: {:?}\n\n", payload)); + broadcaster + .lock() + .expect("Can't lock broadcaster") + .send(msg); + } + }); + + Ok(()) +} + impl ToRedisArgs for &Planet { fn write_redis_args(&self, out: &mut W) where diff --git a/mongodb-redis/src/services.rs b/mongodb-redis/src/services.rs index 80a2c39..a57c01a 100644 --- a/mongodb-redis/src/services.rs +++ b/mongodb-redis/src/services.rs @@ -1,13 +1,10 @@ use std::str::FromStr; -use actix_web::web::Bytes; use chrono::{Timelike, Utc}; use log::debug; use mongodb::bson::oid::ObjectId; use redis::aio::ConnectionManager; -use redis::{AsyncCommands, Client, FromRedisValue, Value}; -use tokio::sync::mpsc::{self, Receiver}; -use tokio_stream::StreamExt; +use redis::{AsyncCommands, Client, Value}; use crate::db::MongoDbClient; use crate::dto::PlanetMessage; @@ -19,7 +16,7 @@ const PLANET_KEY_PREFIX: &str = "planet"; const IMAGE_KEY_PREFIX: &str = "image"; const RATE_LIMIT_KEY_PREFIX: &str = "rate_limit"; const MAX_REQUESTS_PER_MINUTE: u64 = 10; -const NEW_PLANETS_CHANNEL_NAME: &str = "new_planets"; +pub const NEW_PLANETS_CHANNEL_NAME: &str = "new_planets"; #[derive(Clone)] pub struct PlanetService { @@ -158,37 +155,6 @@ impl PlanetService { } } - pub async fn get_new_planets_stream( - &self, - ) -> Result>, CustomError> { - let (tx, rx) = mpsc::channel::>(100); - - tx.send(Ok(Bytes::from("data: Connected\n\n"))) - .await - .expect("Can't send a message to the stream"); - - let mut pubsub_con = self - .redis_client - .get_async_connection() - .await? - .into_pubsub(); - pubsub_con.subscribe(NEW_PLANETS_CHANNEL_NAME).await?; - - tokio::spawn(async move { - while let Some(msg) = pubsub_con.on_message().next().await { - let payload = msg.get_payload().expect("Can't get payload of message"); - let payload: String = FromRedisValue::from_redis_value(&payload) - .expect("Can't convert from Redis value"); - let msg = Bytes::from(format!("data: Planet created: {:?}\n\n", payload)); - tx.send(Ok(msg)) - .await - .expect("Can't send a message to the stream"); - } - }); - - Ok(rx) - } - fn get_planet_cache_key(&self, planet_id: &str) -> String { format!("{}:{}", PLANET_KEY_PREFIX, planet_id) }