Merge pull request #573 from sebadob/include-quorum-in-health-check

Include quorum in `/health` check

book/src/config/config.md

@@ -408,6 +408,18 @@ CACHE_USERS_LIFESPAN=28800
 # !!! USE WITH CARE !!!
 #MIGRATE_DB_FROM=sqlite:data/rauthy.db
 
+# Defines the time in seconds after which the `/health` endpoint 
+# includes HA quorum checks. The initial delay solves problems 
+# like Kubernetes StatefulSet starts that include the health 
+# endpoint in the scheduling routine. In these cases, the scheduler 
+# will not start other Pods if the first does not become healthy.
+# 
+# This is a chicken-and-egg problem which the delay solves.
+# There is usually no need to adjust this value.
+#
+# default: 30
+#HEALTH_CHECK_DELAY_SECS=30
+
 # Disables the housekeeping schedulers (default: false)
 #SCHED_DISABLE=true
 

rauthy.cfg

@@ -373,6 +373,18 @@ HQL_SECRET_API=SuperSecureSecret1337
 #MIGRATE_DB_FROM=sqlite:data/rauthy.db
 #MIGRATE_DB_FROM=postgresql://postgres:123SuperSafe@localhost:5432/rauthy
 
+# Defines the time in seconds after which the `/health` endpoint
+# includes HA quorum checks. The initial delay solves problems
+# like Kubernetes StatefulSet starts that include the health 
+# endpoint in the scheduling routine. In these cases, the scheduler
+# will not start other Pods if the first does not become healthy.
+#
+# This is a chicken-and-egg problem which the delay solves.
+# There is usually no need to adjust this value.
+#
+# default: 30
+#HEALTH_CHECK_DELAY_SECS=30
+
 # Disables the housekeeping schedulers (default: false)
 #SCHED_DISABLE=true
 

src/api/src/generic.rs

@@ -10,8 +10,8 @@ use rauthy_api_types::generic::{
     PasswordPolicyRequest, PasswordPolicyResponse, SearchParams, SearchParamsType,
 };
 use rauthy_common::constants::{
-    APPLICATION_JSON, HEADER_ALLOW_ALL_ORIGINS, HEADER_HTML, IDX_LOGIN_TIME, RAUTHY_VERSION,
-    SUSPICIOUS_REQUESTS_BLACKLIST, SUSPICIOUS_REQUESTS_LOG,
+    APPLICATION_JSON, APP_START, HEADER_ALLOW_ALL_ORIGINS, HEADER_HTML, HEALTH_CHECK_DELAY_SECS,
+    IDX_LOGIN_TIME, RAUTHY_VERSION, SUSPICIOUS_REQUESTS_BLACKLIST, SUSPICIOUS_REQUESTS_LOG,
 };
 use rauthy_common::utils::real_ip_from_req;
 use rauthy_error::ErrorResponse;
@@ -47,9 +47,9 @@ use rauthy_models::templates::{
 use rauthy_service::{encryption, suspicious_request_block};
 use semver::Version;
 use std::borrow::Cow;
-use std::ops::Add;
+use std::ops::{Add, Sub};
 use std::str::FromStr;
-use tracing::{error, warn};
+use tracing::{error, info, warn};
 
 #[get("/")]
 pub async fn get_index(
@@ -614,18 +614,26 @@ pub async fn post_update_language(
 )]
 #[get("/health")]
 pub async fn get_health(data: web::Data<AppState>) -> impl Responder {
-    let db_healthy = is_db_alive(&data.db).await;
-    let cache_healthy = DB::client().is_healthy_cache().await.is_ok();
-
-    let body = HealthResponse {
-        db_healthy,
-        cache_healthy,
-    };
-
-    if db_healthy && cache_healthy {
-        HttpResponse::Ok().json(body)
+    if Utc::now().sub(*APP_START).num_seconds() < *HEALTH_CHECK_DELAY_SECS as i64 {
+        info!("Early health check within the HEALTH_CHECK_DELAY_SECS timeframe - returning true");
+        HttpResponse::Ok().json(HealthResponse {
+            db_healthy: true,
+            cache_healthy: true,
+        })
     } else {
-        HttpResponse::InternalServerError().json(body)
+        let db_healthy = is_db_alive(&data.db).await;
+        let cache_healthy = DB::client().is_healthy_cache().await.is_ok();
+
+        let body = HealthResponse {
+            db_healthy,
+            cache_healthy,
+        };
+
+        if db_healthy && cache_healthy {
+            HttpResponse::Ok().json(body)
+        } else {
+            HttpResponse::InternalServerError().json(body)
+        }
     }
 }
 

src/bin/src/main.rs

@@ -8,7 +8,9 @@ use actix_web::{middleware, web, App, HttpServer};
 use actix_web_prom::PrometheusMetricsBuilder;
 use cryptr::EncKeys;
 use prometheus::Registry;
-use rauthy_common::constants::{RAUTHY_VERSION, SWAGGER_UI_EXTERNAL, SWAGGER_UI_INTERNAL};
+use rauthy_common::constants::{
+    APP_START, RAUTHY_VERSION, SWAGGER_UI_EXTERNAL, SWAGGER_UI_INTERNAL,
+};
 use rauthy_common::password_hasher;
 use rauthy_common::utils::UseDummyAddress;
 use rauthy_handlers::openapi::ApiDoc;
@@ -78,7 +80,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
 
     let log_level = setup_logging();
 
-    info!("Starting Rauthy v{}", RAUTHY_VERSION);
+    info!("{} - Starting Rauthy v{}", *APP_START, RAUTHY_VERSION);
     info!("Log Level set to '{}'", log_level);
     if test_mode {
         info!("Application started in Integration Test Mode");

src/common/src/constants.rs

@@ -1,6 +1,7 @@
 use crate::utils::build_trusted_proxies;
 use crate::DbType;
 use actix_web::http::Uri;
+use chrono::{DateTime, Utc};
 use lazy_static::lazy_static;
 use regex::Regex;
 use std::env;
@@ -16,7 +17,6 @@ pub enum CookieMode {
 }
 
 pub const RAUTHY_VERSION: &str = env!("CARGO_PKG_VERSION");
-
 pub const CONTENT_TYPE_WEBP: &str = "image/webp";
 pub const HEADER_DPOP_NONCE: &str = "DPoP-Nonce";
 pub const HEADER_ALLOW_ALL_ORIGINS: (&str, &str) = ("access-control-allow-origin", "*");
@@ -88,6 +88,8 @@ pub const IDX_WEBAUTHN: &str = "webauthn_";
 
 // TODO drop `lazy_static` and use rust 1.80 built-in features
 lazy_static! {
+    pub static ref APP_START: DateTime<Utc> = Utc::now();
+
     pub static ref CACHE_TTL_AUTH_CODE: Option<i64> = Some(300 + *WEBAUTHN_REQ_EXP as i64);
     pub static ref CACHE_TTL_DEVICE_CODE: Option<i64> = Some(*DEVICE_GRANT_CODE_LIFETIME as i64);
     pub static ref CACHE_TTL_DYN_CLIENT: Option<i64> = Some(*DYN_CLIENT_RATE_LIMIT_SEC as i64);
@@ -102,9 +104,10 @@ lazy_static! {
     pub static ref CACHE_TTL_WEBAUTHN_DATA: Option<i64> = Some(*WEBAUTHN_DATA_EXP as i64);
 
     pub static ref RAUTHY_ADMIN_ROLE: String = "rauthy_admin".to_string();
+
     pub static ref DATABASE_URL: String = env::var("DATABASE_URL").expect("DATABASE_URL is not set");
     pub static ref DB_TYPE: DbType = DbType::from_str(&DATABASE_URL).unwrap();
-    pub static ref ROLE_ADMIN: String = "rauthy_admin".to_string();
+
     pub static ref DEV_MODE: bool = env::var("DEV_MODE")
         .unwrap_or_else(|_| String::from("false"))
         .parse::<bool>()
@@ -187,6 +190,11 @@ lazy_static! {
     pub static ref AUTH_HEADER_MFA: String = env::var("AUTH_HEADER_MFA")
         .unwrap_or_else(|_| String::from("x-forwarded-user-mfa"));
 
+     pub static ref HEALTH_CHECK_DELAY_SECS: u16 = env::var("HEALTH_CHECK_DELAY_SECS")
+        .unwrap_or_else(|_| "30".to_string())
+        .parse::<u16>()
+        .expect("HEALTH_CHECK_DELAY_SECS cannot be parsed to u16 - bad format");
+
     pub static ref COOKIE_MODE: CookieMode = {
         let var = env::var("COOKIE_MODE").unwrap_or_else(|_| "host".to_string());
         match var.as_str() {

src/models/src/entity/principal.rs

@@ -1,7 +1,7 @@
 use crate::entity::api_keys::{AccessGroup, AccessRights, ApiKey};
 use crate::entity::sessions::{Session, SessionState};
 use actix_web::{web, HttpRequest};
-use rauthy_common::constants::{ADMIN_FORCE_MFA, ROLE_ADMIN};
+use rauthy_common::constants::{ADMIN_FORCE_MFA, RAUTHY_ADMIN_ROLE};
 use rauthy_error::{ErrorResponse, ErrorResponseType};
 use tracing::trace;
 
@@ -45,7 +45,7 @@ impl Principal {
 
     #[inline(always)]
     pub fn is_admin(&self) -> bool {
-        self.roles.contains(&*ROLE_ADMIN)
+        self.roles.contains(&*RAUTHY_ADMIN_ROLE)
     }
 
     #[inline(always)]