From 47a10ea14e941d8c7f30f3c21fbf0605db75fd2d Mon Sep 17 00:00:00 2001 From: christn Date: Fri, 16 Aug 2024 16:31:50 +0800 Subject: [PATCH] feat(phoenix): Add delay_update_monitor.rs --- ...6cd024d3f687645d6605b30e58b402ab2b5b4.json | 20 +++++++++ src/phoenix/auction_analysis_monitor.rs | 26 ++--------- src/phoenix/delay_update_monitor.rs | 43 +++++++++++++++++++ src/phoenix/env.rs | 7 +++ src/phoenix/mod.rs | 4 ++ src/phoenix/util.rs | 22 ++++++++++ 6 files changed, 99 insertions(+), 23 deletions(-) create mode 100644 .sqlx/query-a9e4c026ffc88f66c7710a25d796cd024d3f687645d6605b30e58b402ab2b5b4.json create mode 100644 src/phoenix/delay_update_monitor.rs create mode 100644 src/phoenix/util.rs diff --git a/.sqlx/query-a9e4c026ffc88f66c7710a25d796cd024d3f687645d6605b30e58b402ab2b5b4.json b/.sqlx/query-a9e4c026ffc88f66c7710a25d796cd024d3f687645d6605b30e58b402ab2b5b4.json new file mode 100644 index 0000000..7dcd5ea --- /dev/null +++ b/.sqlx/query-a9e4c026ffc88f66c7710a25d796cd024d3f687645d6605b30e58b402ab2b5b4.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT MAX(latest_header_slot)\n FROM header_delay_updates\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "a9e4c026ffc88f66c7710a25d796cd024d3f687645d6605b30e58b402ab2b5b4" +} diff --git a/src/phoenix/auction_analysis_monitor.rs b/src/phoenix/auction_analysis_monitor.rs index 56d7e60..3a7b43d 100644 --- a/src/phoenix/auction_analysis_monitor.rs +++ b/src/phoenix/auction_analysis_monitor.rs @@ -1,26 +1,12 @@ use anyhow::Result; -use chrono::{DateTime, Utc}; -use lazy_static::lazy_static; use sqlx::PgPool; use tracing::debug; -use crate::{ - env::{Network, ToNetwork}, - phoenix::{Alarm, AlarmType}, -}; +use super::util::get_current_slot; -use super::env::APP_CONFIG; - -lazy_static! { - static ref GENESIS_TIMESTAMP: DateTime = { - match &APP_CONFIG.env.to_network() { - Network::Mainnet => "2020-12-01T12:00:23Z".parse().unwrap(), - Network::Holesky => "2023-09-28T12:00:00Z".parse().unwrap(), - } - }; -} +use crate::phoenix::{Alarm, AlarmType}; -const SECONDS_PER_SLOT: u8 = 12; +use super::env::APP_CONFIG; async fn get_latest_auction_analysis_slot(mev_pool: &PgPool) -> anyhow::Result { sqlx::query_scalar!( @@ -39,12 +25,6 @@ async fn get_latest_auction_analysis_slot(mev_pool: &PgPool) -> anyhow::Result Result { - let now = Utc::now(); - let seconds_since_genesis: u32 = (now - *GENESIS_TIMESTAMP).num_seconds().try_into()?; - Ok(seconds_since_genesis / SECONDS_PER_SLOT as u32) -} - pub async fn run_auction_analysis_monitor(mev_pool: &PgPool, alarm: &mut Alarm) -> Result<()> { let latest_slot = get_latest_auction_analysis_slot(mev_pool).await?; let current_slot = get_current_slot()?; diff --git a/src/phoenix/delay_update_monitor.rs b/src/phoenix/delay_update_monitor.rs new file mode 100644 index 0000000..af44720 --- /dev/null +++ b/src/phoenix/delay_update_monitor.rs @@ -0,0 +1,43 @@ +use anyhow::Result; +use sqlx::PgPool; +use tracing::debug; + +use super::util::get_current_slot; +use crate::phoenix::{Alarm, AlarmType}; + +use super::env::APP_CONFIG; + +async fn get_latest_header_delay_updates_slot(mev_pool: &PgPool) -> anyhow::Result { + sqlx::query_scalar!( + r#" + SELECT MAX(latest_header_slot) + FROM header_delay_updates + "#, + ) + .fetch_one(mev_pool) + .await + .map(|max| { + max.expect("No maximum slot found from header_delay_updates") + .try_into() + .expect("Maximum slot is negative") + }) + .map_err(Into::into) +} + +pub async fn run_header_delay_updates_monitor(mev_pool: &PgPool, alarm: &mut Alarm) -> Result<()> { + let latest_slot = get_latest_header_delay_updates_slot(mev_pool).await?; + let current_slot = get_current_slot()?; + let slot_lag = current_slot - latest_slot; + debug!( + "header_delay_updates is {:} slots behind current slot", + slot_lag + ); + if slot_lag > APP_CONFIG.max_header_delay_updates_slot_lag { + let message = format!( + "header_delay_updates is {:} slots behind the current slot", + slot_lag + ); + alarm.fire(&message, &AlarmType::Telegram).await; + } + Ok(()) +} diff --git a/src/phoenix/env.rs b/src/phoenix/env.rs index 9f4ea61..f7107f2 100644 --- a/src/phoenix/env.rs +++ b/src/phoenix/env.rs @@ -35,6 +35,13 @@ pub struct AppConfig { pub unsynced_nodes_threshold_og_alert: usize, #[serde(default = "default_max_auction_analysis_slot_lag")] pub max_auction_analysis_slot_lag: u32, + #[serde(default = "default_max_header_delay_updates_slot_lag")] + pub max_header_delay_updates_slot_lag: u32, +} + +fn default_max_header_delay_updates_slot_lag() -> u32 { + // Should be configured considering the configured schedule for the update cron job + 60 } fn default_max_auction_analysis_slot_lag() -> u32 { diff --git a/src/phoenix/mod.rs b/src/phoenix/mod.rs index 8ed9ab4..bb9155a 100644 --- a/src/phoenix/mod.rs +++ b/src/phoenix/mod.rs @@ -2,10 +2,12 @@ mod alerts; mod auction_analysis_monitor; mod checkpoint; mod consensus_node; +mod delay_update_monitor; mod demotion_monitor; mod env; mod inclusion_monitor; mod promotion_monitor; +mod util; mod validation_node; use std::{ @@ -36,6 +38,7 @@ use self::{ SendAlert, }, auction_analysis_monitor::run_auction_analysis_monitor, + delay_update_monitor::run_header_delay_updates_monitor, demotion_monitor::run_demotion_monitor, inclusion_monitor::{run_inclusion_monitor, LokiClient}, promotion_monitor::run_promotion_monitor, @@ -285,6 +288,7 @@ async fn run_ops_monitors() -> Result<()> { run_inclusion_monitor(&relay_pool, &mev_pool, &canonical_horizon, &loki_client).await?; run_promotion_monitor(&relay_pool, &mev_pool, &canonical_horizon).await?; run_auction_analysis_monitor(&mev_pool, &mut auction_analysis_alarm).await?; + run_header_delay_updates_monitor(&mev_pool, &mut auction_analysis_alarm).await?; tokio::time::sleep(Duration::minutes(1).to_std().unwrap()).await; } } diff --git a/src/phoenix/util.rs b/src/phoenix/util.rs new file mode 100644 index 0000000..6f56517 --- /dev/null +++ b/src/phoenix/util.rs @@ -0,0 +1,22 @@ +use super::env::APP_CONFIG; +use crate::env::{Network, ToNetwork}; +use anyhow::Result; +use chrono::{DateTime, Utc}; +use lazy_static::lazy_static; + +lazy_static! { + static ref GENESIS_TIMESTAMP: DateTime = { + match &APP_CONFIG.env.to_network() { + Network::Mainnet => "2020-12-01T12:00:23Z".parse().unwrap(), + Network::Holesky => "2023-09-28T12:00:00Z".parse().unwrap(), + } + }; +} + +const SECONDS_PER_SLOT: u8 = 12; + +pub fn get_current_slot() -> Result { + let now = Utc::now(); + let seconds_since_genesis: u32 = (now - *GENESIS_TIMESTAMP).num_seconds().try_into()?; + Ok(seconds_since_genesis / SECONDS_PER_SLOT as u32) +}