feat(phoenix): Add delay_update_monitor.rs

src/phoenix/auction_analysis_monitor.rs

@@ -1,26 +1,12 @@
 use anyhow::Result;
-use chrono::{DateTime, Utc};
-use lazy_static::lazy_static;
 use sqlx::PgPool;
 use tracing::debug;
 
-use crate::{
-    env::{Network, ToNetwork},
-    phoenix::{Alarm, AlarmType},
-};
+use super::util::get_current_slot;
 
-use super::env::APP_CONFIG;
-
-lazy_static! {
-    static ref GENESIS_TIMESTAMP: DateTime<Utc> = {
-        match &APP_CONFIG.env.to_network() {
-            Network::Mainnet => "2020-12-01T12:00:23Z".parse().unwrap(),
-            Network::Holesky => "2023-09-28T12:00:00Z".parse().unwrap(),
-        }
-    };
-}
+use crate::phoenix::{Alarm, AlarmType};
 
-const SECONDS_PER_SLOT: u8 = 12;
+use super::env::APP_CONFIG;
 
 async fn get_latest_auction_analysis_slot(mev_pool: &PgPool) -> anyhow::Result<u32> {
     sqlx::query_scalar!(
@@ -39,12 +25,6 @@ async fn get_latest_auction_analysis_slot(mev_pool: &PgPool) -> anyhow::Result<u
     .map_err(Into::into)
 }
 
-fn get_current_slot() -> Result<u32> {
-    let now = Utc::now();
-    let seconds_since_genesis: u32 = (now - *GENESIS_TIMESTAMP).num_seconds().try_into()?;
-    Ok(seconds_since_genesis / SECONDS_PER_SLOT as u32)
-}
-
 pub async fn run_auction_analysis_monitor(mev_pool: &PgPool, alarm: &mut Alarm) -> Result<()> {
     let latest_slot = get_latest_auction_analysis_slot(mev_pool).await?;
     let current_slot = get_current_slot()?;

src/phoenix/delay_update_monitor.rs

@@ -0,0 +1,43 @@
+use anyhow::Result;
+use sqlx::PgPool;
+use tracing::debug;
+
+use super::util::get_current_slot;
+use crate::phoenix::{Alarm, AlarmType};
+
+use super::env::APP_CONFIG;
+
+async fn get_latest_header_delay_updates_slot(mev_pool: &PgPool) -> anyhow::Result<u32> {
+    sqlx::query_scalar!(
+        r#"
+        SELECT MAX(latest_header_slot)
+        FROM header_delay_updates
+        "#,
+    )
+    .fetch_one(mev_pool)
+    .await
+    .map(|max| {
+        max.expect("No maximum slot found from header_delay_updates")
+            .try_into()
+            .expect("Maximum slot is negative")
+    })
+    .map_err(Into::into)
+}
+
+pub async fn run_header_delay_updates_monitor(mev_pool: &PgPool, alarm: &mut Alarm) -> Result<()> {
+    let latest_slot = get_latest_header_delay_updates_slot(mev_pool).await?;
+    let current_slot = get_current_slot()?;
+    let slot_lag = current_slot - latest_slot;
+    debug!(
+        "header_delay_updates is {:} slots behind current slot",
+        slot_lag
+    );
+    if slot_lag > APP_CONFIG.max_header_delay_updates_slot_lag {
+        let message = format!(
+            "header_delay_updates is {:} slots behind the current slot",
+            slot_lag
+        );
+        alarm.fire(&message, &AlarmType::Telegram).await;
+    }
+    Ok(())
+}

src/phoenix/env.rs

@@ -35,6 +35,13 @@ pub struct AppConfig {
     pub unsynced_nodes_threshold_og_alert: usize,
     #[serde(default = "default_max_auction_analysis_slot_lag")]
     pub max_auction_analysis_slot_lag: u32,
+    #[serde(default = "default_max_header_delay_updates_slot_lag")]
+    pub max_header_delay_updates_slot_lag: u32,
+}
+
+fn default_max_header_delay_updates_slot_lag() -> u32 {
+    // Should be configured considering the configured schedule for the update cron job
+    60
 }
 
 fn default_max_auction_analysis_slot_lag() -> u32 {

src/phoenix/mod.rs

@@ -2,10 +2,12 @@ mod alerts;
 mod auction_analysis_monitor;
 mod checkpoint;
 mod consensus_node;
+mod delay_update_monitor;
 mod demotion_monitor;
 mod env;
 mod inclusion_monitor;
 mod promotion_monitor;
+mod util;
 mod validation_node;
 
 use std::{
@@ -36,6 +38,7 @@ use self::{
         SendAlert,
     },
     auction_analysis_monitor::run_auction_analysis_monitor,
+    delay_update_monitor::run_header_delay_updates_monitor,
     demotion_monitor::run_demotion_monitor,
     inclusion_monitor::{run_inclusion_monitor, LokiClient},
     promotion_monitor::run_promotion_monitor,
@@ -285,6 +288,7 @@ async fn run_ops_monitors() -> Result<()> {
         run_inclusion_monitor(&relay_pool, &mev_pool, &canonical_horizon, &loki_client).await?;
         run_promotion_monitor(&relay_pool, &mev_pool, &canonical_horizon).await?;
         run_auction_analysis_monitor(&mev_pool, &mut auction_analysis_alarm).await?;
+        run_header_delay_updates_monitor(&mev_pool, &mut auction_analysis_alarm).await?;
         tokio::time::sleep(Duration::minutes(1).to_std().unwrap()).await;
     }
 }

src/phoenix/util.rs

@@ -0,0 +1,22 @@
+use super::env::APP_CONFIG;
+use crate::env::{Network, ToNetwork};
+use anyhow::Result;
+use chrono::{DateTime, Utc};
+use lazy_static::lazy_static;
+
+lazy_static! {
+    static ref GENESIS_TIMESTAMP: DateTime<Utc> = {
+        match &APP_CONFIG.env.to_network() {
+            Network::Mainnet => "2020-12-01T12:00:23Z".parse().unwrap(),
+            Network::Holesky => "2023-09-28T12:00:00Z".parse().unwrap(),
+        }
+    };
+}
+
+const SECONDS_PER_SLOT: u8 = 12;
+
+pub fn get_current_slot() -> Result<u32> {
+    let now = Utc::now();
+    let seconds_since_genesis: u32 = (now - *GENESIS_TIMESTAMP).num_seconds().try_into()?;
+    Ok(seconds_since_genesis / SECONDS_PER_SLOT as u32)
+}