ref(tokio): Limit blocking tokio pool to 150 threads

relay-server/src/service.rs

@@ -1,6 +1,7 @@
 use std::convert::Infallible;
 use std::fmt;
 use std::sync::Arc;
+use std::time::Duration;
 
 use crate::metrics::{MetricOutcomes, MetricStats};
 use crate::services::stats::RelayStats;
@@ -77,6 +78,15 @@ pub fn create_runtime(name: &str, threads: usize) -> Runtime {
     tokio::runtime::Builder::new_multi_thread()
         .thread_name(name)
         .worker_threads(threads)
+        // Relay uses `spawn_blocking` only for Redis connections within the project
+        // cache, those should never exceed 100 concurrent connections.
+        // We can limit the maximum amount of threads here, we've seen that Tokio
+        // expands this pool very very aggressively and basically never shrinks it
+        // which leads to a massive resource waste.
+        .max_blocking_threads(150)
+        // As with the maximum amount of threads used by the runtime, we want
+        // to encourge the runtime to terminate blocking threads again.
+        .thread_keep_alive(Duration::from_secs(1))
         .enable_all()
         .build()
         .unwrap()